From 358d95d72cf39238fcf385ebc88bbbc7191d31bd Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 19 Apr 2022 16:54:31 +0300 Subject: [PATCH 001/234] Upgrade dependencies for Django 3.x upgrade Also converted to using pyproject.toml --- .gitignore | 2 + pyproject.toml | 8 ++ requirements/base.in | 14 +- requirements/base.pip | 323 +++++++++++++----------------------------- setup.cfg | 112 +++++++++++++++ setup.py | 120 +--------------- 6 files changed, 229 insertions(+), 350 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 09b294b470..7011f00d57 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ tags .bash_history .inputrc + +.eggs diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..fdad2c0aba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools >= 48", + "setuptools_scm[toml] >= 4, <6", + "setuptools_scm_git_archive", + "wheel >= 0.29.0", +] +build-backend = 'setuptools.build_meta' diff --git a/requirements/base.in b/requirements/base.in index 5b015c0d3c..24480fac29 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,10 +2,10 @@ -e . # installed from Git --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client --e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc +#-e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest +#-e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest +#-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router +#-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip +#-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient +#-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client +#-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index 8cc148e79b..0e9e81d503 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -1,89 +1,63 @@ # -# This file is autogenerated by pip-compile with python 3.6 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile --output-file=requirements/base.pip requirements/base.in # --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest - # via -r requirements/base.in --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router - # via -r requirements/base.in --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client - # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc - # via -r requirements/base.in --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip - # via -r requirements/base.in --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest - # via -r requirements/base.in --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient - # via -r requirements/base.in alabaster==0.7.12 # via sphinx -amqp==5.0.6 +amqp==5.1.1 # via kombu -analytics-python==1.3.1 +analytics-python==1.4.0 # via onadata appoptics-metrics==5.1.0 # via onadata -argparse==1.4.0 - # via unittest2 -attrs==21.2.0 - # via jsonschema +asgiref==3.5.0 + # via django +async-timeout==4.0.2 + # via redis babel==2.9.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery -boto3==1.17.74 - # via tabulator -botocore==1.20.74 - # via - # boto3 - # s3transfer -cached-property==1.5.2 - # via tableschema -celery==5.0.5 +celery==5.2.6 # via onadata -certifi==2020.12.5 +certifi==2021.10.8 # via requests -cffi==1.14.5 +cffi==1.15.0 # via cryptography -chardet==4.0.0 - # via - # datapackage - # requests - # tabulator -click==7.1.2 +charset-normalizer==2.0.12 + # via requests +click==8.1.2 # via # celery # click-didyoumean # click-plugins # click-repl - # datapackage - # tableschema - # tabulator -click-didyoumean==0.0.3 +click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.1.6 +click-repl==0.2.0 # via celery -cryptography==3.4.7 +cryptography==36.0.2 # via # jwcrypto # onadata - # pyjwt -datapackage==1.15.2 - # via pyfloip defusedxml==0.7.1 - # via djangorestframework-xml -deprecated==1.2.12 - # via onadata -dict2xml==1.7.0 + # via + # djangorestframework-xml + # pyxform +deprecated==1.2.13 + # via + # jwcrypto + # onadata + # redis +dict2xml==1.7.1 # via onadata -django==2.2.23 +django==3.2.13 # via # django-cors-headers # django-debug-toolbar @@ -98,56 +72,53 @@ django==2.2.23 # djangorestframework # djangorestframework-guardian # djangorestframework-jsonapi - # jsonfield - # ona-oidc # onadata -django-activity-stream==0.10.0 +django-activity-stream==1.4.0 # via onadata -django-cors-headers==3.7.0 +django-cors-headers==3.11.0 # via onadata -django-debug-toolbar==3.2.1 +django-debug-toolbar==3.2.4 # via onadata -django-filter==2.4.0 +django-filter==21.1 # via onadata -django-guardian==2.3.0 +django-guardian==2.4.0 # via # djangorestframework-guardian # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==1.5.0 +django-oauth-toolkit==1.7.1 # via onadata -django-ordered-model==3.4.3 +django-ordered-model==3.5 # via onadata django-query-builder==2.0.1 # via onadata -django-redis==5.0.0 +django-redis==5.2.0 # via onadata -django-registration-redux==2.9 +django-registration-redux==2.10 # via onadata -django-render-block==0.8.1 +django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.9 +django-reversion==5.0.0 # via onadata -django-taggit==1.4.0 +django-taggit==2.1.0 # via onadata -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via onadata -djangorestframework==3.12.4 +djangorestframework==3.13.1 # via # djangorestframework-csv # djangorestframework-gis # djangorestframework-guardian # djangorestframework-jsonapi - # ona-oidc # onadata djangorestframework-csv==2.1.1 # via onadata -djangorestframework-gis==0.17 +djangorestframework-gis==0.18 # via onadata djangorestframework-guardian==0.3.0 # via onadata -djangorestframework-jsonapi==4.2.0 +djangorestframework-jsonapi==5.0.0 # via onadata djangorestframework-jsonp==1.0.2 # via onadata @@ -155,82 +126,42 @@ djangorestframework-xml==2.0.0 # via onadata docutils==0.17.1 # via sphinx -dpath==2.0.1 +dpath==2.0.6 # via onadata elaphe3==0.2.0 # via onadata et-xmlfile==1.1.0 # via openpyxl -flake8==3.9.2 +flake8==4.0.1 # via onadata -fleming==0.6.0 +fleming==0.7.0 # via django-query-builder -formencode==1.3.1 - # via pyxform -future==0.18.2 - # via python-json2xlsclient geojson==2.5.0 # via onadata -greenlet==1.1.0 - # via sqlalchemy httmock==1.4.0 # via onadata -httplib2==0.19.1 - # via - # oauth2client - # onadata -idna==2.10 +httplib2==0.20.4 + # via onadata +idna==3.3 # via requests -ijson==3.1.4 - # via tabulator -imagesize==1.2.0 +imagesize==1.3.0 # via sphinx -importlib-metadata==4.8.1 - # via - # flake8 - # jsonpickle - # jsonschema - # kombu - # markdown - # sqlalchemy inflection==0.5.1 # via djangorestframework-jsonapi -isodate==0.6.0 - # via tableschema -jinja2==2.11.3 +jinja2==3.1.1 # via sphinx -jmespath==0.10.0 - # via - # boto3 - # botocore -jsonfield==0.9.23 +jsonpickle==2.1.0 # via onadata -jsonlines==2.0.0 - # via tabulator -jsonpickle==2.0.0 - # via onadata -jsonpointer==2.1 - # via datapackage -jsonschema==3.2.0 - # via - # datapackage - # tableschema -jwcrypto==0.8 +jwcrypto==1.0 # via django-oauth-toolkit -kombu==5.0.2 +kombu==5.2.4 # via celery -linear-tsv==1.1.0 - # via tabulator -linecache2==1.0.0 - # via traceback2 -lxml==4.6.3 +lxml==4.8.0 # via onadata -markdown==3.3.4 +markdown==3.3.6 # via onadata -markupsafe==1.1.1 - # via - # jinja2 - # sphinx +markupsafe==2.1.1 + # via jinja2 mccabe==0.6.1 # via flake8 mock==4.0.3 @@ -241,194 +172,134 @@ monotonic==1.6 # via analytics-python nose==1.3.7 # via django-nose -numpy==1.19.5 +numpy==1.22.3 # via onadata -oauthlib==3.1.0 +oauthlib==3.2.0 # via django-oauth-toolkit -openpyxl==3.0.7 +openpyxl==3.0.9 # via # onadata - # tabulator -packaging==20.9 - # via sphinx -paho-mqtt==1.5.1 + # pyxform +packaging==21.3 + # via + # redis + # sphinx +paho-mqtt==1.6.1 # via onadata -pillow==8.2.0 +pillow==9.1.0 # via # elaphe3 # onadata -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 - # via onadata -pyasn1==0.4.8 - # via - # oauth2client - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via oauth2client -pycodestyle==2.7.0 +pycodestyle==2.8.0 # via flake8 -pycparser==2.20 +pycparser==2.21 # via cffi -pyflakes==2.3.1 +pyflakes==2.4.0 # via flake8 -pygments==2.9.0 +pygments==2.11.2 # via sphinx -pyjwt[crypto]==2.1.0 - # via - # ona-oidc - # onadata +pyjwt==2.3.0 + # via onadata pylibmc==1.6.1 # via onadata -pymongo==3.11.4 +pymongo==4.1.1 # via onadata -pyparsing==2.4.7 +pyparsing==3.0.8 # via # httplib2 # packaging -pyrsistent==0.17.3 - # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # analytics-python - # botocore # fleming # onadata - # tableschema python-memcached==1.59 # via onadata -pytz==2021.1 +pytz==2022.1 # via # babel # celery # django # django-query-builder + # djangorestframework # fleming # onadata -pyxform==1.5.0 - # via - # onadata - # pyfloip +pyxform==1.9.0 + # via onadata raven==6.10.0 # via onadata recaptcha-client==1.0.6 # via onadata -redis==3.5.3 +redis==4.2.2 # via django-redis -requests==2.25.1 +requests==2.27.1 # via # analytics-python - # datapackage # django-oauth-toolkit # httmock - # ona-oidc # onadata - # python-json2xlsclient # requests-mock # sphinx - # tableschema - # tabulator -requests-mock==1.9.2 - # via onadata -rfc3986==1.5.0 - # via tableschema -rsa==4.7.2 - # via oauth2client -s3transfer==0.4.2 - # via boto3 +requests-mock==1.9.3 + # via onadata savreaderwriter==3.4.2 # via onadata -simplejson==3.17.2 +simplejson==3.17.6 # via onadata six==1.16.0 # via # analytics-python # appoptics-metrics # click-repl - # datapackage - # django-oauth-toolkit # django-query-builder - # django-templated-email # djangorestframework-csv - # isodate - # jsonschema - # linear-tsv - # oauth2client # python-dateutil # python-memcached # requests-mock - # tableschema - # tabulator - # unittest2 -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx -sphinx==4.0.1 +sphinx==4.5.0 # via onadata sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy==1.4.15 - # via tabulator -sqlparse==0.4.1 +sqlparse==0.4.2 # via # django # django-debug-toolbar -tableschema==1.20.2 - # via datapackage -tabulator==1.53.5 - # via - # datapackage - # tableschema -traceback2==1.4.0 - # via unittest2 -typing-extensions==3.10.0.2 - # via importlib-metadata unicodecsv==0.14.1 # via - # datapackage # djangorestframework-csv # onadata - # pyxform - # tableschema - # tabulator -unittest2==1.1.0 - # via pyxform -urllib3==1.26.4 - # via - # botocore - # requests -uwsgi==2.0.19.1 +urllib3==1.26.9 + # via requests +uwsgi==2.0.20 # via onadata vine==5.0.0 # via # amqp # celery + # kombu wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.12.1 +wrapt==1.14.0 # via deprecated -xlrd==1.2.0 +xlrd==2.0.1 # via # onadata # pyxform - # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 # via onadata -zipp==3.6.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..1b794f9ed5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,112 @@ +[metadata] +name = onadata +description = Collect Analyze and Share Data +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://github.com/onaio/onadata +author = Ona Systems Inc +author_email = support@ona.io +license = Copyright (c) 2022 Ona Systems Inc All rights reserved +license_file = LICENSE +classifiers = + Development Status :: 5 - Production/Stable + Programming Language :: Python :: 3.8 +project_urls = + Documentation = https://api.ona.io/api + Source = https://github.com/onaio/onadata + Tracker = https://github.com/onaio/onadata/issues + +[options] +packages = find: +platforms = any +install_requires = + Django>=2.2.20,<4 + django-guardian + django-registration-redux + django-templated-email + django-reversion + django-filter + django-nose + django-ordered-model + #generic relation + django-query-builder + celery + #cors + django-cors-headers + django-debug-toolbar + #oauth2 support + django-oauth-toolkit + #oauth2client + jsonpickle + #jwt + PyJWT + #captcha + recaptcha-client + #API support + djangorestframework + djangorestframework-csv + djangorestframework-gis + djangorestframework-guardian + djangorestframework-jsonapi + djangorestframework-jsonp + djangorestframework-xml + #geojson + geojson + #tagging + django-taggit + #database + #psycopg2-binary>2.7.1 + pymongo + #sms support + dict2xml + lxml + #pyxform + pyxform + #spss + savreaderwriter + #tests + mock + httmock + #memcached support + pylibmc + python-memcached + #XML Instance API utility + xmltodict + #docs + sphinx + Markdown + #others + unicodecsv + xlrd + xlwt + openpyxl + dpath + elaphe3 + httplib2 + modilabs-python-utils + numpy + Pillow + python-dateutil + pytz + requests + requests-mock + simplejson + uwsgi + flake8 + raven + django-activity-stream + paho-mqtt + cryptography + #Monitoring + analytics-python + appoptics-metrics + # Deprecation tagging + deprecated + # Redis cache + django-redis +python_requires = >= 3.8 +setup_requires = + setuptools_scm + +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index cd762c2947..0c93a07ecf 100644 --- a/setup.py +++ b/setup.py @@ -14,121 +14,7 @@ https://ona.io https://opendatakit.org """ +import setuptools -from setuptools import setup, find_packages - -import onadata - -setup( - name="onadata", - version=onadata.__version__, - description="Collect Analyze and Share Data!", - author="Ona Systems Inc", - author_email="support@ona.io", - license="Copyright (c) 2014 Ona Systems Inc All rights reserved.", - project_urls={ - 'Source': 'https://github.com/onaio/onadata', - }, - packages=find_packages(exclude=['docs', 'tests']), - install_requires=[ - "Django>=2.2.20,<3", - "django-guardian", - "django-registration-redux", - "django-templated-email", - "django-reversion", - "django-filter", - "django-nose", - "django-ordered-model", - # generic relation - "django-query-builder", - "celery", - # cors - "django-cors-headers", - "django-debug-toolbar", - # oauth2 support - "django-oauth-toolkit", - # "oauth2client", - "jsonpickle", - # jwt - "PyJWT", - # captcha - "recaptcha-client", - # API support - "djangorestframework", - "djangorestframework-csv", - "djangorestframework-gis", - "djangorestframework-guardian", - "djangorestframework-jsonapi", - "djangorestframework-jsonp", - "djangorestframework-xml", - # geojson - "geojson", - # tagging - "django-taggit", - # database - "psycopg2>2.7.1", - "pymongo", - # sms support - "dict2xml", - "lxml", - # pyxform - "pyxform", - # spss - "savreaderwriter", - # tests - "mock", - "httmock", - # JSON data type support, keeping it around for previous migration - "jsonfield<1.0", - # memcached support - "pylibmc", - "python-memcached", - # XML Instance API utility - "xmltodict", - # docs - "sphinx", - "Markdown", - # others - "unicodecsv", - "xlrd", - "xlwt", - "openpyxl", - "dpath", - "elaphe3", - "httplib2", - "modilabs-python-utils", - "numpy", - "Pillow", - "python-dateutil", - "pytz", - "requests", - "requests-mock", - "simplejson", - "uwsgi", - "flake8", - "raven", - "django-activity-stream", - "paho-mqtt", - "cryptography", - # Monitoring - "analytics-python", - "appoptics-metrics", - # Deprecation tagging - "deprecated", - # Redis cache - "django-redis", - ], - dependency_links=[ - 'https://github.com/onaio/python-digest/tarball/3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-digest/tarball/eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-multidb-router/tarball/9cf0a0c6c9f796e5bd14637fafeb5a1a5507ed37#egg=django-multidb-router', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/floip-py/tarball/3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=floip', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/python-json2xlsclient/tarball/62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/oauth2client/tarball/75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client', # noqa pylint: disable=line-too-long - ], - extras_require={ - ':python_version=="2.7"': [ - 'functools32>=3.2.3-2' - ] - } -) +if __name__ == "__main__": + setuptools.setup() From f0b918585f3901538d08ef67c19f830bd06fc750 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 01:18:55 +0300 Subject: [PATCH 002/234] Update basestring references --- onadata/apps/api/tasks.py | 25 +- .../tests/viewsets/test_attachment_viewset.py | 287 +- onadata/apps/api/viewsets/dataview_viewset.py | 227 +- .../apps/api/viewsets/user_profile_viewset.py | 238 +- .../management/commands/import_instances.py | 51 +- onadata/apps/logger/models/instance.py | 368 ++- onadata/apps/logger/models/widget.py | 117 +- .../apps/logger/tests/models/test_xform.py | 50 +- onadata/apps/main/models/meta_data.py | 247 +- .../apps/main/tests/test_form_enter_data.py | 27 +- onadata/apps/sms_support/tools.py | 358 +- onadata/libs/models/signals.py | 16 +- onadata/libs/serializers/chart_serializer.py | 23 +- onadata/libs/serializers/fields/json_field.py | 8 +- .../serializers/organization_serializer.py | 108 +- .../serializers/user_profile_serializer.py | 319 +- .../libs/tests/utils/test_export_builder.py | 2896 +++++++++-------- onadata/libs/utils/chart_tools.py | 290 +- onadata/libs/utils/common_tools.py | 48 +- onadata/libs/utils/csv_builder.py | 719 ++-- onadata/libs/utils/dict_tools.py | 41 +- onadata/libs/utils/qrcode.py | 35 +- onadata/libs/utils/string.py | 13 +- onadata/libs/utils/viewer_tools.py | 143 +- onadata/settings/common.py | 479 ++- 25 files changed, 3830 insertions(+), 3303 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index b8e1f4ff2f..6f965e7943 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -7,7 +7,6 @@ from django.core.files.storage import default_storage from django.contrib.auth.models import User from django.utils.datastructures import MultiValueDict -from past.builtins import basestring from onadata.apps.api import tools from onadata.libs.utils.email import send_generic_email @@ -26,7 +25,7 @@ def recreate_tmp_file(name, path, mime_type): def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): try: files = MultiValueDict() - files[u'xls_file'] = default_storage.open(file_data.get('path')) + files["xls_file"] = default_storage.open(file_data.get("path")) owner = User.objects.get(id=owner_id) if owner_id == user_id: @@ -34,7 +33,7 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): else: user = User.objects.get(id=user_id) survey = tools.do_publish_xlsform(user, post_data, files, owner) - default_storage.delete(file_data.get('path')) + default_storage.delete(file_data.get("path")) if isinstance(survey, XForm): return {"pk": survey.pk} @@ -46,13 +45,13 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): self.retry(exc=exc, countdown=1) else: error_message = ( - u'Service temporarily unavailable, please try to ' - 'publish the form again' + "Service temporarily unavailable, please try to " + "publish the form again" ) else: error_message = str(sys.exc_info()[1]) - return {u'error': error_message} + return {"error": error_message} @app.task() @@ -66,23 +65,23 @@ def delete_xform_async(xform_id, user_id): @app.task() def delete_user_async(): """Delete inactive user accounts""" - users = User.objects.filter(active=False, - username__contains="deleted-at", - email__contains="deleted-at") + users = User.objects.filter( + active=False, username__contains="deleted-at", email__contains="deleted-at" + ) for user in users: user.delete() def get_async_status(job_uuid): - """ Gets progress status or result """ + """Gets progress status or result""" if not job_uuid: - return {u'error': u'Empty job uuid'} + return {"error": "Empty job uuid"} job = AsyncResult(job_uuid) result = job.result or job.state - if isinstance(result, basestring): - return {'JOB_STATUS': result} + if isinstance(result, str): + return {"JOB_STATUS": result} return result diff --git a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py index 0ff13097c8..67249eb069 100644 --- a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py @@ -1,11 +1,9 @@ import os -from past.builtins import basestring from django.utils import timezone -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models.attachment import Attachment @@ -18,22 +16,15 @@ def attachment_url(attachment, suffix=None): path = get_attachment_url(attachment, suffix) - return u'http://testserver{}'.format(path) + return "http://testserver{}".format(path) class TestAttachmentViewSet(TestAbstractViewSet): - def setUp(self): super(TestAttachmentViewSet, self).setUp() - self.retrieve_view = AttachmentViewSet.as_view({ - 'get': 'retrieve' - }) - self.list_view = AttachmentViewSet.as_view({ - 'get': 'list' - }) - self.count_view = AttachmentViewSet.as_view({ - 'get': 'count' - }) + self.retrieve_view = AttachmentViewSet.as_view({"get": "retrieve"}) + self.list_view = AttachmentViewSet.as_view({"get": "list"}) + self.count_view = AttachmentViewSet.as_view({"get": "count"}) self._publish_xls_form_to_project() @@ -43,36 +34,36 @@ def test_retrieve_view(self): pk = self.attachment.pk data = { - 'url': 'http://testserver/api/v1/media/%s' % pk, - 'field_xpath': 'image1', - 'download_url': attachment_url(self.attachment), - 'small_download_url': attachment_url(self.attachment, 'small'), - 'medium_download_url': attachment_url(self.attachment, 'medium'), - 'id': pk, - 'xform': self.xform.pk, - 'instance': self.attachment.instance.pk, - 'mimetype': self.attachment.mimetype, - 'filename': self.attachment.media_file.name + "url": "http://testserver/api/v1/media/%s" % pk, + "field_xpath": "image1", + "download_url": attachment_url(self.attachment), + "small_download_url": attachment_url(self.attachment, "small"), + "medium_download_url": attachment_url(self.attachment, "medium"), + "id": pk, + "xform": self.xform.pk, + "instance": self.attachment.instance.pk, + "mimetype": self.attachment.mimetype, + "filename": self.attachment.media_file.name, } - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, dict)) self.assertEqual(response.data, data) # file download - filename = data['filename'] - ext = filename[filename.rindex('.') + 1:] - request = self.factory.get('/', **self.extra) + filename = data["filename"] + ext = filename[filename.rindex(".") + 1 :] + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk, format=ext) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, 'image/jpeg') + self.assertEqual(response.content_type, "image/jpeg") self.attachment.instance.xform.deleted_at = timezone.now() self.attachment.instance.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) @@ -83,41 +74,46 @@ def test_attachment_pagination(self): self._submit_transport_instance_w_attachment() self.assertEqual(self.response.status_code, 201) filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "image2", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='image/jpeg', - extension='JPG', + mimetype="image/jpeg", + extension="JPG", name=filename, - media_file=media_file) + media_file=media_file, + ) # not using pagination params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # valid page and page_size - request = self.factory.get( - '/', data={"page": 1, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 1, "page_size": 1}, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) # invalid page type - request = self.factory.get('/', data={"page": "invalid"}, **self.extra) + request = self.factory.get("/", data={"page": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size type - request = self.factory.get('/', data={"page_size": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) @@ -125,14 +121,13 @@ def test_attachment_pagination(self): # invalid page and page_size types request = self.factory.get( - '/', data={"page": "invalid", "page_size": "invalid"}, - **self.extra) + "/", data={"page": "invalid", "page_size": "invalid"}, **self.extra + ) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size - request = self.factory.get( - '/', data={"page": 4, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 4, "page_size": 1}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -143,16 +138,16 @@ def test_retrieve_and_list_views_with_anonymous_user(self): pk = self.attachment.pk xform_id = self.attachment.instance.xform.id - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -160,24 +155,24 @@ def test_retrieve_and_list_views_with_anonymous_user(self): xform.shared_data = True xform.save() - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 200) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 200) def test_list_view(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) @@ -186,7 +181,7 @@ def test_list_view(self): self.attachment.instance.deleted_at = timezone.now() self.attachment.instance.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 0) @@ -194,16 +189,16 @@ def test_list_view(self): def test_data_list_with_xform_in_delete_async(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), initial_count - 1) @@ -211,188 +206,194 @@ def test_data_list_with_xform_in_delete_async(self): def test_list_view_filter_by_xform(self): self._submit_transport_instance_w_attachment() - data = { - 'xform': self.xform.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"xform": self.xform.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['xform'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["xform"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['xform'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["xform"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_instance(self): self._submit_transport_instance_w_attachment() - data = { - 'instance': self.attachment.instance.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"instance": self.attachment.instance.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['instance'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["instance"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['instance'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["instance"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_attachment_type(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'video2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "video2", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='video/mp4', - extension='MP4', + mimetype="video/mp4", + extension="MP4", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='application/pdf', - extension='PDF', + mimetype="application/pdf", + extension="PDF", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='text/plain', - extension='TXT', + mimetype="text/plain", + extension="TXT", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='audio/mp3', - extension='MP3', + mimetype="audio/mp3", + extension="MP3", name=filename, - media_file=media_file) + media_file=media_file, + ) data = {} - request = self.factory.get('/', data, **self.extra) + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 5) # Apply image Filter - data['type'] = 'image' - request = self.factory.get('/', data, **self.extra) + data["type"] = "image" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'image/jpeg') + self.assertEqual(response.data[0]["mimetype"], "image/jpeg") # Apply audio filter - data['type'] = 'audio' - request = self.factory.get('/', data, **self.extra) + data["type"] = "audio" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'audio/mp3') + self.assertEqual(response.data[0]["mimetype"], "audio/mp3") # Apply video filter - data['type'] = 'video' - request = self.factory.get('/', data, **self.extra) + data["type"] = "video" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'video/mp4') + self.assertEqual(response.data[0]["mimetype"], "video/mp4") # Apply file filter - data['type'] = 'document' - request = self.factory.get('/', data, **self.extra) + data["type"] = "document" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) - self.assertEqual(response.data[0]["mimetype"], 'application/pdf') - self.assertEqual(response.data[1]["mimetype"], 'text/plain') + self.assertEqual(response.data[0]["mimetype"], "application/pdf") + self.assertEqual(response.data[1]["mimetype"], "text/plain") def test_direct_image_link(self): self._submit_transport_instance_w_attachment() - data = { - 'filename': self.attachment.media_file.name - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": self.attachment.media_file.name} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) - data['filename'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["filename"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) - data['filename'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["filename"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) def test_direct_image_link_uppercase(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - self.attachment.media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + self.attachment.media_file = django_file(path, "image2", "image/jpeg") self.attachment.name = filename self.attachment.save() filename = self.attachment.media_file.name file_base, file_extension = os.path.splitext(filename) - data = { - 'filename': file_base + file_extension.upper() - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": file_base + file_extension.upper()} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) def test_total_count(self): self._submit_transport_instance_w_attachment() xform_id = self.attachment.instance.xform.id - request = self.factory.get( - '/count', data={"xform": xform_id}, **self.extra) + request = self.factory.get("/count", data={"xform": xform_id}, **self.extra) response = self.count_view(request) - self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data["count"], 1) def test_returned_attachments_is_based_on_form_permissions(self): # Create a form and make submissions with attachments self._submit_transport_instance_w_attachment() formid = self.xform.pk - request = self.factory.get( - '/', data={"xform": formid}, **self.extra) + request = self.factory.get("/", data={"xform": formid}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -403,13 +404,11 @@ def test_returned_attachments_is_based_on_form_permissions(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) ShareXForm(self.xform, user_dave.username, EditorRole.name) - auth_extra = { - 'HTTP_AUTHORIZATION': f'Token {user_dave.auth_token.key}' - } + auth_extra = {"HTTP_AUTHORIZATION": f"Token {user_dave.auth_token.key}"} # Dave user should not be able to view attachments for # submissions which they did not submit - request = self.factory.get('/', data={"xform": formid}, **auth_extra) + request = self.factory.get("/", data={"xform": formid}, **auth_extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) # Ensure no submissions are returned for the User diff --git a/onadata/apps/api/viewsets/dataview_viewset.py b/onadata/apps/api/viewsets/dataview_viewset.py index a26474a78c..03538645e0 100644 --- a/onadata/apps/api/viewsets/dataview_viewset.py +++ b/onadata/apps/api/viewsets/dataview_viewset.py @@ -1,5 +1,3 @@ -from past.builtins import basestring - from django.db.models.signals import post_delete, post_save from django.http import Http404, HttpResponseBadRequest @@ -16,8 +14,7 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.data_view import DataView from onadata.apps.viewer.models.export import Export -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.renderers import renderers @@ -25,16 +22,22 @@ from onadata.libs.serializers.dataview_serializer import DataViewSerializer from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils import common_tags -from onadata.libs.utils.api_export_tools import (custom_response_handler, - export_async_export_response, - include_hxl_row, - process_async_export, - response_for_format) -from onadata.libs.utils.cache_tools import (PROJECT_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.chart_tools import (get_chart_data_for_field, - get_field_from_field_name) +from onadata.libs.utils.api_export_tools import ( + custom_response_handler, + export_async_export_response, + include_hxl_row, + process_async_export, + response_for_format, +) +from onadata.libs.utils.cache_tools import ( + PROJECT_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.chart_tools import ( + get_chart_data_for_field, + get_field_from_field_name, +) from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl @@ -42,12 +45,12 @@ def get_form_field_chart_url(url, field): - return u'%s?field_name=%s' % (url, field) + return "%s?field_name=%s" % (url, field) -class DataViewViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - ModelViewSet): +class DataViewViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): """ A simple ViewSet for viewing and editing DataViews. """ @@ -55,7 +58,7 @@ class DataViewViewSet(AuthenticateHeaderMixin, queryset = DataView.objects.select_related() serializer_class = DataViewSerializer permission_classes = [DataViewViewsetPermissions] - lookup_field = 'pk' + lookup_field = "pk" renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ renderers.XLSRenderer, renderers.XLSXRenderer, @@ -66,48 +69,53 @@ class DataViewViewSet(AuthenticateHeaderMixin, ] def get_serializer_class(self): - if self.action == 'data': + if self.action == "data": serializer_class = JsonDataSerializer else: serializer_class = self.serializer_class return serializer_class - @action(methods=['GET'], detail=True) - def data(self, request, format='json', **kwargs): + @action(methods=["GET"], detail=True) + def data(self, request, format="json", **kwargs): """Retrieve the data from the xform using this dataview""" start = request.GET.get("start") limit = request.GET.get("limit") count = request.GET.get("count") sort = request.GET.get("sort") query = request.GET.get("query") - export_type = self.kwargs.get('format', request.GET.get("format")) + export_type = self.kwargs.get("format", request.GET.get("format")) self.object = self.get_object() - if export_type is None or export_type in ['json', 'debug']: - data = DataView.query_data(self.object, start, limit, - str_to_bool(count), sort=sort, - filter_query=query) - if 'error' in data: - raise ParseError(data.get('error')) + if export_type is None or export_type in ["json", "debug"]: + data = DataView.query_data( + self.object, + start, + limit, + str_to_bool(count), + sort=sort, + filter_query=query, + ) + if "error" in data: + raise ParseError(data.get("error")) serializer = self.get_serializer(data, many=True) return Response(serializer.data) else: - return custom_response_handler(request, self.object.xform, query, - export_type, - dataview=self.object) + return custom_response_handler( + request, self.object.xform, query, export_type, dataview=self.object + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): params = request.query_params - job_uuid = params.get('job_uuid') - export_type = params.get('format') - include_hxl = params.get('include_hxl', False) - include_labels = params.get('include_labels', False) - include_labels_only = params.get('include_labels_only', False) + job_uuid = params.get("job_uuid") + export_type = params.get("format") + include_hxl = params.get("include_hxl", False) + include_labels = params.get("include_labels", False) + include_labels_only = params.get("include_labels_only", False) query = params.get("query") dataview = self.get_object() xform = dataview.xform @@ -121,99 +129,98 @@ def export_async(self, request, *args, **kwargs): if include_hxl is not None: include_hxl = str_to_bool(include_hxl) - remove_group_name = params.get('remove_group_name', False) - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + remove_group_name = params.get("remove_group_name", False) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl and include_hxl: - include_hxl = include_hxl_row( - dataview.columns, list(columns_with_hxl) - ) + include_hxl = include_hxl_row(dataview.columns, list(columns_with_hxl)) options = { - 'remove_group_name': remove_group_name, - 'dataview_pk': dataview.pk, - 'include_hxl': include_hxl, - 'include_labels': include_labels, - 'include_labels_only': include_labels_only + "remove_group_name": remove_group_name, + "dataview_pk": dataview.pk, + "include_hxl": include_hxl, + "include_labels": include_labels, + "include_labels_only": include_labels_only, } if query: - options.update({'query': query}) + options.update({"query": query}) if job_uuid: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": export_id = job.result export = Export.objects.get(id=export_id) resp = export_async_export_response(request, export) else: - resp = { - 'job_status': job.state - } + resp = {"job_status": job.state} else: - resp = process_async_export(request, xform, export_type, - options=options) + resp = process_async_export(request, xform, export_type, options=options) - return Response(data=resp, - status=status.HTTP_202_ACCEPTED, - content_type="application/json") + return Response( + data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" + ) - @action(methods=['GET'], detail=True) - def form(self, request, format='json', **kwargs): + @action(methods=["GET"], detail=True) + def form(self, request, format="json", **kwargs): dataview = self.get_object() xform = dataview.xform - if format not in ['json', 'xml', 'xls']: - return HttpResponseBadRequest('400 BAD REQUEST', - content_type='application/json', - status=400) + if format not in ["json", "xml", "xls"]: + return HttpResponseBadRequest( + "400 BAD REQUEST", content_type="application/json", status=400 + ) filename = xform.id_string + "." + format response = response_for_format(xform, format=format) - response['Content-Disposition'] = 'attachment; filename=' + filename + response["Content-Disposition"] = "attachment; filename=" + filename return response - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def form_details(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform - serializer = XFormSerializer(xform, context={'request': request}) + serializer = XFormSerializer(xform, context={"request": request}) - return Response(data=serializer.data, - content_type="application/json") + return Response(data=serializer.data, content_type="application/json") - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def charts(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform serializer = self.get_serializer(dataview) - field_name = request.query_params.get('field_name') - field_xpath = request.query_params.get('field_xpath') - fmt = kwargs.get('format', request.accepted_renderer.format) - group_by = request.query_params.get('group_by') + field_name = request.query_params.get("field_name") + field_xpath = request.query_params.get("field_xpath") + fmt = kwargs.get("format", request.accepted_renderer.format) + group_by = request.query_params.get("group_by") if field_name: field = get_field_from_field_name(field_name, xform) - field_xpath = field_name if isinstance(field, basestring) \ - else field.get_abbreviated_xpath() + field_xpath = ( + field_name if isinstance(field, str) else field.get_abbreviated_xpath() + ) - if field_xpath and field_xpath not in dataview.columns and \ - field_xpath not in [common_tags.SUBMISSION_TIME, - common_tags.SUBMITTED_BY, - common_tags.DURATION]: - raise Http404( - "Field %s does not not exist on the dataview" % field_name) + if ( + field_xpath + and field_xpath not in dataview.columns + and field_xpath + not in [ + common_tags.SUBMISSION_TIME, + common_tags.SUBMITTED_BY, + common_tags.DURATION, + ] + ): + raise Http404("Field %s does not not exist on the dataview" % field_name) if field_name or field_xpath: data = get_chart_data_for_field( - field_name, xform, fmt, group_by, field_xpath, - data_view=dataview + field_name, xform, fmt, group_by, field_xpath, data_view=dataview ) - return Response(data, template_name='chart_detail.html') + return Response(data, template_name="chart_detail.html") - if fmt != 'json' and field_name is None: + if fmt != "json" and field_name is None: raise ParseError("Not supported") data = serializer.data @@ -221,14 +228,18 @@ def charts(self, request, *args, **kwargs): for field in xform.survey_elements: field_xpath = field.get_abbreviated_xpath() if field_xpath in dataview.columns: - url = reverse('dataviews-charts', kwargs={'pk': dataview.pk}, - request=request, format=fmt) + url = reverse( + "dataviews-charts", + kwargs={"pk": dataview.pk}, + request=request, + format=fmt, + ) field_url = get_form_field_chart_url(url, field.name) data["fields"][field.name] = field_url return Response(data) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def xls_export(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform @@ -236,40 +247,38 @@ def xls_export(self, request, *args, **kwargs): token = None export_type = "xls" query = request.query_params.get("query", {}) - meta = request.GET.get('meta') + meta = request.GET.get("meta") - return custom_response_handler(request, - xform, - query, - export_type, - token, - meta, - dataview) + return custom_response_handler( + request, xform, query, export_type, token, meta, dataview + ) def destroy(self, request, *args, **kwargs): dataview = self.get_object() user = request.user dataview.soft_delete(user) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, dataview.project.pk)) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, dataview.project.pk)) return Response(status=status.HTTP_204_NO_CONTENT) -def dataview_post_save_callback(sender, instance=None, created=False, - **kwargs): - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) +def dataview_post_save_callback(sender, instance=None, created=False, **kwargs): + safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) def dataview_post_delete_callback(sender, instance, **kwargs): if instance.project: - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - instance.project.pk)) + safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) -post_save.connect(dataview_post_save_callback, - sender=DataView, - dispatch_uid='dataview_post_save_callback') +post_save.connect( + dataview_post_save_callback, + sender=DataView, + dispatch_uid="dataview_post_save_callback", +) -post_delete.connect(dataview_post_delete_callback, - sender=DataView, - dispatch_uid='dataview_post_delete_callback') +post_delete.connect( + dataview_post_delete_callback, + sender=DataView, + dispatch_uid="dataview_post_delete_callback", +) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 3aa8b18e12..16db8e1c42 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -7,7 +7,6 @@ import json from future.moves.urllib.parse import urlencode -from past.builtins import basestring # pylint: disable=redefined-builtin from django.conf import settings from django.core.cache import cache @@ -33,28 +32,27 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.libs.utils.email import (get_verification_email_data, - get_verification_url) -from onadata.libs.utils.cache_tools import (safe_delete, - CHANGE_PASSWORD_ATTEMPTS, - LOCKOUT_CHANGE_PASSWORD_USER, - USER_PROFILE_PREFIX) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url +from onadata.libs.utils.cache_tools import ( + safe_delete, + CHANGE_PASSWORD_ATTEMPTS, + LOCKOUT_CHANGE_PASSWORD_USER, + USER_PROFILE_PREFIX, +) from onadata.libs import filters from onadata.libs.utils.user_auth import invalidate_and_regen_tokens -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin -from onadata.libs.serializers.monthly_submissions_serializer import \ - MonthlySubmissionsSerializer -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileSerializer +from onadata.libs.serializers.monthly_submissions_serializer import ( + MonthlySubmissionsSerializer, +) +from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name -LOCKOUT_TIME = getattr(settings, 'LOCKOUT_TIME', 1800) -MAX_CHANGE_PASSWORD_ATTEMPTS = getattr( - settings, 'MAX_CHANGE_PASSWORD_ATTEMPTS', 10) +LOCKOUT_TIME = getattr(settings, "LOCKOUT_TIME", 1800) +MAX_CHANGE_PASSWORD_ATTEMPTS = getattr(settings, "MAX_CHANGE_PASSWORD_ATTEMPTS", 10) def replace_key_value(lookup, new_value, expected_dict): @@ -101,31 +99,30 @@ def serializer_from_settings(): def set_is_email_verified(profile, is_email_verified): - profile.metadata.update({'is_email_verified': is_email_verified}) + profile.metadata.update({"is_email_verified": is_email_verified}) profile.save() def check_user_lockout(request): username = request.user.username - lockout = cache.get('{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username)) + lockout = cache.get("{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username)) response_obj = { - 'error': 'Too many password reset attempts, Try again in {} minutes'} + "error": "Too many password reset attempts, Try again in {} minutes" + } if lockout: - time_locked_out = \ - datetime.datetime.now() - datetime.datetime.strptime( - lockout, '%Y-%m-%dT%H:%M:%S') - remaining_time = round( - (LOCKOUT_TIME - - time_locked_out.seconds) / 60) - response = response_obj['error'].format(remaining_time) + time_locked_out = datetime.datetime.now() - datetime.datetime.strptime( + lockout, "%Y-%m-%dT%H:%M:%S" + ) + remaining_time = round((LOCKOUT_TIME - time_locked_out.seconds) / 60) + response = response_obj["error"].format(remaining_time) return response def change_password_attempts(request): """Track number of login attempts made by user within a specified amount - of time""" + of time""" username = request.user.username - password_attempts = '{}{}'.format(CHANGE_PASSWORD_ATTEMPTS, username) + password_attempts = "{}{}".format(CHANGE_PASSWORD_ATTEMPTS, username) attempts = cache.get(password_attempts) if attempts: @@ -133,9 +130,10 @@ def change_password_attempts(request): attempts = cache.get(password_attempts) if attempts >= MAX_CHANGE_PASSWORD_ATTEMPTS: cache.set( - '{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username), - datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - LOCKOUT_TIME) + "{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username), + datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + LOCKOUT_TIME, + ) if check_user_lockout(request): return check_user_lockout(request) @@ -147,29 +145,32 @@ def change_password_attempts(request): class UserProfileViewSet( - AuthenticateHeaderMixin, # pylint: disable=R0901 - CacheControlMixin, - ETagsMixin, - ObjectLookupMixin, - BaseViewset, - ModelViewSet): + AuthenticateHeaderMixin, # pylint: disable=R0901 + CacheControlMixin, + ETagsMixin, + ObjectLookupMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create/Register users. """ - queryset = UserProfile.objects.select_related().filter( - user__is_active=True).exclude( - user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + + queryset = ( + UserProfile.objects.select_related() + .filter(user__is_active=True) + .exclude(user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + ) serializer_class = serializer_from_settings() - lookup_field = 'user' + lookup_field = "user" permission_classes = [UserProfilePermissions] filter_backends = (filters.UserProfileFilter, OrderingFilter) - ordering = ('user__username', ) + ordering = ("user__username",) def get_object(self, queryset=None): """Lookup user profile by pk or username""" if self.kwargs.get(self.lookup_field, None) is None: - raise ParseError( - 'Expected URL keyword argument `%s`.' % self.lookup_field) + raise ParseError("Expected URL keyword argument `%s`." % self.lookup_field) if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -180,7 +181,7 @@ def get_object(self, queryset=None): if self.lookup_field in serializer.get_fields(): k = serializer.get_fields()[self.lookup_field] if isinstance(k, serializers.HyperlinkedRelatedField): - lookup_field = '%s__%s' % (self.lookup_field, k.lookup_field) + lookup_field = "%s__%s" % (self.lookup_field, k.lookup_field) lookup = self.kwargs[self.lookup_field] filter_kwargs = {lookup_field: lookup} @@ -188,9 +189,9 @@ def get_object(self, queryset=None): try: user_pk = int(lookup) except (TypeError, ValueError): - filter_kwargs = {'%s__iexact' % lookup_field: lookup} + filter_kwargs = {"%s__iexact" % lookup_field: lookup} else: - filter_kwargs = {'user__pk': user_pk} + filter_kwargs = {"user__pk": user_pk} obj = get_object_or_404(queryset, **filter_kwargs) @@ -200,84 +201,72 @@ def get_object(self, queryset=None): return obj def update(self, request, *args, **kwargs): - """ Update user in cache and db""" - username = kwargs.get('user') - response = super(UserProfileViewSet, self)\ - .update(request, *args, **kwargs) - cache.set(f'{USER_PROFILE_PREFIX}{username}', response.data) + """Update user in cache and db""" + username = kwargs.get("user") + response = super(UserProfileViewSet, self).update(request, *args, **kwargs) + cache.set(f"{USER_PROFILE_PREFIX}{username}", response.data) return response def retrieve(self, request, *args, **kwargs): - """ Get user profile from cache or db """ - username = kwargs.get('user') - cached_user = cache.get(f'{USER_PROFILE_PREFIX}{username}') + """Get user profile from cache or db""" + username = kwargs.get("user") + cached_user = cache.get(f"{USER_PROFILE_PREFIX}{username}") if cached_user: return Response(cached_user) - response = super(UserProfileViewSet, self)\ - .retrieve(request, *args, **kwargs) + response = super(UserProfileViewSet, self).retrieve(request, *args, **kwargs) return response def create(self, request, *args, **kwargs): - """ Create and cache user profile """ - response = super(UserProfileViewSet, self)\ - .create(request, *args, **kwargs) + """Create and cache user profile""" + response = super(UserProfileViewSet, self).create(request, *args, **kwargs) profile = response.data - user_name = profile.get('username') - cache.set(f'{USER_PROFILE_PREFIX}{user_name}', profile) + user_name = profile.get("username") + cache.set(f"{USER_PROFILE_PREFIX}{user_name}", profile) return response - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def change_password(self, request, *args, **kwargs): # noqa """ Change user's password. """ # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") user_profile = self.get_object() - current_password = request.data.get('current_password', None) - new_password = request.data.get('new_password', None) + current_password = request.data.get("current_password", None) + new_password = request.data.get("new_password", None) lock_out = check_user_lockout(request) - response_obj = { - 'error': 'Invalid password. You have {} attempts left.'} + response_obj = {"error": "Invalid password. You have {} attempts left."} if new_password: if not lock_out: if user_profile.user.check_password(current_password): - data = { - 'username': user_profile.user.username - } + data = {"username": user_profile.user.username} metadata = user_profile.metadata or {} - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() user_profile.user.set_password(new_password) user_profile.metadata = metadata user_profile.user.save() user_profile.save() - data.update(invalidate_and_regen_tokens( - user=user_profile.user)) + data.update(invalidate_and_regen_tokens(user=user_profile.user)) - return Response( - status=status.HTTP_200_OK, data=data) + return Response(status=status.HTTP_200_OK, data=data) response = change_password_attempts(request) if isinstance(response, int): - limits_remaining = \ - MAX_CHANGE_PASSWORD_ATTEMPTS - response - response = response_obj['error'].format( - limits_remaining) - return Response(data=response, - status=status.HTTP_400_BAD_REQUEST) + limits_remaining = MAX_CHANGE_PASSWORD_ATTEMPTS - response + response = response_obj["error"].format(limits_remaining) + return Response(data=response, status=status.HTTP_400_BAD_REQUEST) return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) def partial_update(self, request, *args, **kwargs): profile = self.get_object() metadata = profile.metadata or {} - if request.data.get('overwrite') == 'false': - if isinstance(request.data.get('metadata'), basestring): - metadata_items = json.loads( - request.data.get('metadata')).items() + if request.data.get("overwrite") == "false": + if isinstance(request.data.get("metadata"), str): + metadata_items = json.loads(request.data.get("metadata")).items() else: - metadata_items = request.data.get('metadata').items() + metadata_items = request.data.get("metadata").items() for key, value in metadata_items: if check_if_key_exists(key, metadata): @@ -289,26 +278,24 @@ def partial_update(self, request, *args, **kwargs): profile.save() return Response(data=profile.metadata, status=status.HTTP_200_OK) - return super(UserProfileViewSet, self).partial_update( - request, *args, **kwargs) + return super(UserProfileViewSet, self).partial_update(request, *args, **kwargs) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def monthly_submissions(self, request, *args, **kwargs): - """ Get the total number of submissions for a user """ + """Get the total number of submissions for a user""" # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") profile = self.get_object() - month_param = self.request.query_params.get('month', None) - year_param = self.request.query_params.get('year', None) + month_param = self.request.query_params.get("month", None) + year_param = self.request.query_params.get("year", None) # check if parameters are valid if month_param: - if not month_param.isdigit() or \ - int(month_param) not in range(1, 13): - raise ValidationError(u'Invalid month provided as parameter') + if not month_param.isdigit() or int(month_param) not in range(1, 13): + raise ValidationError("Invalid month provided as parameter") if year_param: if not year_param.isdigit() or len(year_param) != 4: - raise ValidationError(u'Invalid year provided as parameter') + raise ValidationError("Invalid year provided as parameter") # Use query parameter values for month and year # if none, use the current month and year @@ -316,12 +303,16 @@ def monthly_submissions(self, request, *args, **kwargs): month = month_param if month_param else now.month year = year_param if year_param else now.year - instance_count = Instance.objects.filter( - xform__user=profile.user, - xform__deleted_at__isnull=True, - date_created__year=year, - date_created__month=month).values('xform__shared').annotate( - num_instances=Count('id')) + instance_count = ( + Instance.objects.filter( + xform__user=profile.user, + xform__deleted_at__isnull=True, + date_created__year=year, + date_created__month=month, + ) + .values("xform__shared") + .annotate(num_instances=Count("id")) + ) serializer = MonthlySubmissionsSerializer(instance_count, many=True) return Response(serializer.data[0]) @@ -333,21 +324,21 @@ def verify_email(self, request, *args, **kwargs): if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - redirect_url = request.query_params.get('redirect_url') - verification_key = request.query_params.get('verification_key') + redirect_url = request.query_params.get("redirect_url") + verification_key = request.query_params.get("verification_key") response_message = _("Missing or invalid verification key") if verification_key: rp = None try: rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + "user", "user__profile" + ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: with use_master: try: rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + "user", "user__profile" + ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: pass @@ -358,17 +349,13 @@ def verify_email(self, request, *args, **kwargs): username = rp.user.username set_is_email_verified(rp.user.profile, True) # Clear profiles cache - safe_delete(f'{USER_PROFILE_PREFIX}{username}') + safe_delete(f"{USER_PROFILE_PREFIX}{username}") - response_data = { - 'username': username, - 'is_email_verified': True - } + response_data = {"username": username, "is_email_verified": True} if redirect_url: query_params_string = urlencode(response_data) - redirect_url = '{}?{}'.format(redirect_url, - query_params_string) + redirect_url = "{}?{}".format(redirect_url, query_params_string) return HttpResponseRedirect(redirect_url) @@ -376,14 +363,14 @@ def verify_email(self, request, *args, **kwargs): return HttpResponseBadRequest(response_message) - @action(methods=['POST'], detail=False) + @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - username = request.data.get('username') - redirect_url = request.data.get('redirect_url') + username = request.data.get("username") + redirect_url = request.data.get("redirect_url") response_message = _("Verification email has NOT been sent") if username: @@ -396,14 +383,17 @@ def send_verification_email(self, request, *args, **kwargs): verification_key = rp.activation_key if verification_key == verified_key_text: - verification_key = (rp.user.registrationprofile. - create_new_activation_key()) + verification_key = ( + rp.user.registrationprofile.create_new_activation_key() + ) verification_url = get_verification_url( - redirect_url, request, verification_key) + redirect_url, request, verification_key + ) email_data = get_verification_email_data( - rp.user.email, rp.user.username, verification_url, request) + rp.user.email, rp.user.username, verification_url, request + ) send_verification_email.delay(**email_data) response_message = _("Verification email has been sent") diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index eab0d33195..ab12285dd6 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -2,29 +2,34 @@ # vim: ai ts=4 sts=4 et sw=5 coding=utf-8 import os -from past.builtins import basestring from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from onadata.apps.logger.import_tools import (import_instances_from_path, - import_instances_from_zip) +from onadata.apps.logger.import_tools import ( + import_instances_from_path, + import_instances_from_zip, +) class Command(BaseCommand): - args = 'username path' - help = ugettext_lazy("Import a zip file, a directory containing zip files " - "or a directory of ODK instances") + args = "username path" + help = ugettext_lazy( + "Import a zip file, a directory containing zip files " + "or a directory of ODK instances" + ) def _log_import(self, results): total_count, success_count, errors = results - self.stdout.write(_( - "Total: %(total)d, Imported: %(imported)d, Errors: " - "%(errors)s\n------------------------------\n") % { - 'total': total_count, 'imported': success_count, - 'errors': errors}) + self.stdout.write( + _( + "Total: %(total)d, Imported: %(imported)d, Errors: " + "%(errors)s\n------------------------------\n" + ) + % {"total": total_count, "imported": success_count, "errors": errors} + ) def handle(self, *args, **kwargs): if len(args) < 2: @@ -32,18 +37,17 @@ def handle(self, *args, **kwargs): username = args[0] path = args[1] is_async = args[2] if len(args) > 2 else False - is_async = True if isinstance(is_async, basestring) and \ - is_async.lower() == 'true' else False + is_async = ( + True if isinstance(is_async, str) and is_async.lower() == "true" else False + ) try: user = User.objects.get(username=username) except User.DoesNotExist: - raise CommandError(_( - "The specified user '%s' does not exist.") % username) + raise CommandError(_("The specified user '%s' does not exist.") % username) # make sure path exists if not os.path.exists(path): - raise CommandError(_( - "The specified path '%s' does not exist.") % path) + raise CommandError(_("The specified path '%s' does not exist.") % path) for dir, subdirs, files in os.walk(path): # check if the dir has an odk directory @@ -51,15 +55,14 @@ def handle(self, *args, **kwargs): # dont walk further down this dir subdirs.remove("odk") self.stdout.write(_("Importing from dir %s..\n") % dir) - results = import_instances_from_path( - dir, user, is_async=is_async - ) + results = import_instances_from_path(dir, user, is_async=is_async) self._log_import(results) for file in files: filepath = os.path.join(path, file) - if os.path.isfile(filepath) and\ - os.path.splitext(filepath)[1].lower() == ".zip": - self.stdout.write(_( - "Importing from zip at %s..\n") % filepath) + if ( + os.path.isfile(filepath) + and os.path.splitext(filepath)[1].lower() == ".zip" + ): + self.stdout.write(_("Importing from zip at %s..\n") % filepath) results = import_instances_from_zip(filepath, user) self._log_import(results) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index ff85e527ce..fdd90df0b9 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -20,44 +20,74 @@ from django.utils import timezone from django.utils.translation import ugettext as _ from future.utils import python_2_unicode_compatible -from past.builtins import basestring # pylint: disable=W0622 from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview from onadata.apps.logger.models.survey_type import SurveyType from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH, XForm -from onadata.apps.logger.xform_instance_parser import (XFormInstanceParser, - clean_and_parse_xml, - get_uuid_from_xml) +from onadata.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + clean_and_parse_xml, + get_uuid_from_xml, +) from onadata.celery import app from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import ( - DATAVIEW_COUNT, IS_ORG, PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, - XFORM_COUNT, XFORM_DATA_VERSIONS, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) + DATAVIEW_COUNT, + IS_ORG, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + XFORM_DATA_VERSIONS, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, - DELETEDAT, DURATION, EDITED, END, GEOLOCATION, ID, LAST_EDITED, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, MONGO_STRFTIME, NOTES, - REVIEW_STATUS, START, STATUS, SUBMISSION_TIME, SUBMITTED_BY, - TAGS, TOTAL_MEDIA, UUID, VERSION, XFORM_ID, XFORM_ID_STRING, - REVIEW_COMMENT, REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + END, + GEOLOCATION, + ID, + LAST_EDITED, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MONGO_STRFTIME, + NOTES, + REVIEW_STATUS, + START, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID, + XFORM_ID_STRING, + REVIEW_COMMENT, + REVIEW_DATE, +) from onadata.libs.utils.dict_tools import get_values_matching_key from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.timing import calculate_duration -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) def get_attachment_url(attachment, suffix=None): - kwargs = {'pk': attachment.pk} - url = u'{}?filename={}'.format( - reverse('files-detail', kwargs=kwargs), - attachment.media_file.name + kwargs = {"pk": attachment.pk} + url = "{}?filename={}".format( + reverse("files-detail", kwargs=kwargs), attachment.media_file.name ) if suffix: - url += u'&suffix={}'.format(suffix) + url += "&suffix={}".format(suffix) return url @@ -66,15 +96,15 @@ def _get_attachments_from_instance(instance): attachments = [] for a in instance.attachments.filter(deleted_at__isnull=True): attachment = dict() - attachment['download_url'] = get_attachment_url(a) - attachment['small_download_url'] = get_attachment_url(a, 'small') - attachment['medium_download_url'] = get_attachment_url(a, 'medium') - attachment['mimetype'] = a.mimetype - attachment['filename'] = a.media_file.name - attachment['name'] = a.name - attachment['instance'] = a.instance.pk - attachment['xform'] = instance.xform.id - attachment['id'] = a.id + attachment["download_url"] = get_attachment_url(a) + attachment["small_download_url"] = get_attachment_url(a, "small") + attachment["medium_download_url"] = get_attachment_url(a, "medium") + attachment["mimetype"] = a.mimetype + attachment["filename"] = a.media_file.name + attachment["name"] = a.name + attachment["instance"] = a.instance.pk + attachment["xform"] = instance.xform.id + attachment["id"] = a.id attachments.append(attachment) return attachments @@ -89,15 +119,17 @@ def _get_tag_or_element_type_xpath(xform, tag): @python_2_unicode_compatible class FormInactiveError(Exception): """Exception class for inactive forms""" + def __str__(self): - return _(u'Form is inactive') + return _("Form is inactive") @python_2_unicode_compatible class FormIsMergedDatasetError(Exception): """Exception class for merged datasets""" + def __str__(self): - return _(u'Submissions are not allowed on merged datasets.') + return _("Submissions are not allowed on merged datasets.") def numeric_checker(string_value): @@ -113,6 +145,7 @@ def numeric_checker(string_value): return string_value + # need to establish id_string of the xform before we run get_dict since # we now rely on data dictionary to parse the xml @@ -120,15 +153,15 @@ def numeric_checker(string_value): def get_id_string_from_xml_str(xml_str): xml_obj = clean_and_parse_xml(xml_str) root_node = xml_obj.documentElement - id_string = root_node.getAttribute(u"id") + id_string = root_node.getAttribute("id") if len(id_string) == 0: # may be hidden in submission/data/id_string - elems = root_node.getElementsByTagName('data') + elems = root_node.getElementsByTagName("data") for data in elems: for child in data.childNodes: - id_string = data.childNodes[0].getAttribute('id') + id_string = data.childNodes[0].getAttribute("id") if len(id_string) > 0: break @@ -144,15 +177,17 @@ def submission_time(): def _update_submission_count_for_today( - form_id: int, incr: bool = True, date_created=None): + form_id: int, incr: bool = True, date_created=None +): # Track submissions made today current_timzone_name = timezone.get_current_timezone_name() current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - date_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}") - count_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}") + datetime(today.year, today.month, today.day) + ).isoformat() + date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}" + count_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}" if not cache.get(date_cache_key) == current_date: cache.set(date_cache_key, current_date, 86400) @@ -176,42 +211,45 @@ def _update_submission_count_for_today( def update_xform_submission_count(instance_id, created): if created: from multidb.pinning import use_master + with use_master: try: - instance = Instance.objects.select_related('xform').only( - 'xform__user_id', 'date_created').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform") + .only("xform__user_id", "date_created") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: # update xform.num_of_submissions cursor = connection.cursor() sql = ( - 'UPDATE logger_xform SET ' - 'num_of_submissions = num_of_submissions + 1, ' - 'last_submission_time = %s ' - 'WHERE id = %s' + "UPDATE logger_xform SET " + "num_of_submissions = num_of_submissions + 1, " + "last_submission_time = %s " + "WHERE id = %s" ) params = [instance.date_created, instance.xform_id] # update user profile.num_of_submissions cursor.execute(sql, params) sql = ( - 'UPDATE main_userprofile SET ' - 'num_of_submissions = num_of_submissions + 1 ' - 'WHERE user_id = %s' + "UPDATE main_userprofile SET " + "num_of_submissions = num_of_submissions + 1 " + "WHERE user_id = %s" ) cursor.execute(sql, [instance.xform.user_id]) # Track submissions made today _update_submission_count_for_today(instance.xform_id) - safe_delete('{}{}'.format( - XFORM_DATA_VERSIONS, instance.xform_id)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform_id)) - safe_delete('{}{}'.format(XFORM_COUNT, instance.xform_id)) + safe_delete("{}{}".format(XFORM_DATA_VERSIONS, instance.xform_id)) + safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform_id)) + safe_delete("{}{}".format(XFORM_COUNT, instance.xform_id)) # Clear project cache - from onadata.apps.logger.models.xform import \ - clear_project_cache + from onadata.apps.logger.models.xform import clear_project_cache + clear_project_cache(instance.xform.project_id) @@ -224,11 +262,10 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): xform.num_of_submissions -= 1 if xform.num_of_submissions < 0: xform.num_of_submissions = 0 - xform.save(update_fields=['num_of_submissions']) + xform.save(update_fields=["num_of_submissions"]) profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=xform.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=xform.user.profile.pk) except profile_qs.model.DoesNotExist: pass else: @@ -239,15 +276,16 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): # Track submissions made today _update_submission_count_for_today( - xform.id, incr=False, date_created=instance.date_created) + xform.id, incr=False, date_created=instance.date_created + ) for a in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: - safe_delete('{}{}'.format(a, xform.project.pk)) + safe_delete("{}{}".format(a, xform.project.pk)) - safe_delete('{}{}'.format(IS_ORG, xform.pk)) - safe_delete('{}{}'.format(XFORM_DATA_VERSIONS, xform.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, xform.pk)) - safe_delete('{}{}'.format(XFORM_COUNT, xform.pk)) + safe_delete("{}{}".format(IS_ORG, xform.pk)) + safe_delete("{}{}".format(XFORM_DATA_VERSIONS, xform.pk)) + safe_delete("{}{}".format(DATAVIEW_COUNT, xform.pk)) + safe_delete("{}{}".format(XFORM_COUNT, xform.pk)) if xform.instances.exclude(geom=None).count() < 1: xform.instances_with_geopoints = False @@ -264,7 +302,7 @@ def save_full_json(instance_id, created): pass else: instance.json = instance.get_full_dict() - instance.save(update_fields=['json']) + instance.save(update_fields=["json"]) @app.task @@ -272,16 +310,19 @@ def update_project_date_modified(instance_id, created): # update the date modified field of the project which will change # the etag value of the projects endpoint try: - instance = Instance.objects.select_related('xform__project').only( - 'xform__project__date_modified').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform__project") + .only("xform__project__date_modified") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: - instance.xform.project.save(update_fields=['date_modified']) + instance.xform.project.save(update_fields=["date_modified"]) def convert_to_serializable_date(date): - if hasattr(date, 'isoformat'): + if hasattr(date, "isoformat"): return date.isoformat() return date @@ -302,22 +343,20 @@ def numeric_converter(self, json_dict, numeric_fields=None): # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): - if isinstance(value, basestring) and key in numeric_fields: + if isinstance(value, str) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): - json_dict[key] = self.numeric_converter( - value, numeric_fields) + json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): - if isinstance(v, basestring) and key in numeric_fields: + if isinstance(v, str) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): - value[k] = self.numeric_converter( - v, numeric_fields) + value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _set_geom(self): @@ -352,22 +391,25 @@ def get_full_dict(self, load_existing=True): doc = self.get_dict() # pylint: disable=no-member if self.id: - doc.update({ - UUID: self.uuid, - ID: self.id, - BAMBOO_DATASET_ID: self.xform.bamboo_dataset, - ATTACHMENTS: _get_attachments_from_instance(self), - STATUS: self.status, - TAGS: list(self.tags.names()), - NOTES: self.get_notes(), - VERSION: self.version, - DURATION: self.get_duration(), - XFORM_ID_STRING: self._parser.get_xform_id_string(), - XFORM_ID: self.xform.pk, - GEOLOCATION: [self.point.y, self.point.x] if self.point - else [None, None], - SUBMITTED_BY: self.user.username if self.user else None - }) + doc.update( + { + UUID: self.uuid, + ID: self.id, + BAMBOO_DATASET_ID: self.xform.bamboo_dataset, + ATTACHMENTS: _get_attachments_from_instance(self), + STATUS: self.status, + TAGS: list(self.tags.names()), + NOTES: self.get_notes(), + VERSION: self.version, + DURATION: self.get_duration(), + XFORM_ID_STRING: self._parser.get_xform_id_string(), + XFORM_ID: self.xform.pk, + GEOLOCATION: [self.point.y, self.point.x] + if self.point + else [None, None], + SUBMITTED_BY: self.user.username if self.user else None, + } + ) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) @@ -380,8 +422,7 @@ def get_full_dict(self, load_existing=True): review = self.get_latest_review() if review: doc[REVIEW_STATUS] = review.status - doc[REVIEW_DATE] = review.date_created.strftime( - MONGO_STRFTIME) + doc[REVIEW_DATE] = review.date_created.strftime(MONGO_STRFTIME) if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() @@ -392,8 +433,7 @@ def get_full_dict(self, load_existing=True): if not self.date_modified: self.date_modified = self.date_created - doc[DATE_MODIFIED] = self.date_modified.strftime( - MONGO_STRFTIME) + doc[DATE_MODIFIED] = self.date_modified.strftime(MONGO_STRFTIME) doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) @@ -402,13 +442,13 @@ def get_full_dict(self, load_existing=True): doc[MEDIA_ALL_RECEIVED] = self.media_all_received edited = False - if hasattr(self, 'last_edited'): + if hasattr(self, "last_edited"): edited = self.last_edited is not None doc[EDITED] = edited - edited and doc.update({ - LAST_EDITED: convert_to_serializable_date(self.last_edited) - }) + edited and doc.update( + {LAST_EDITED: convert_to_serializable_date(self.last_edited)} + ) return doc def _set_parser(self): @@ -417,8 +457,9 @@ def _set_parser(self): self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): - self.survey_type, created = \ - SurveyType.objects.get_or_create(slug=self.get_root_node_name()) + self.survey_type, created = SurveyType.objects.get_or_create( + slug=self.get_root_node_name() + ) def _set_uuid(self): # pylint: disable=no-member, attribute-defined-outside-init @@ -437,24 +478,26 @@ def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() - instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ + instance_dict = ( + self._parser.get_flat_dict_with_attributes() + if flat else self._parser.to_dict() + ) return self.numeric_converter(instance_dict) def get_notes(self): # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] - @deprecated(version='2.5.3', - reason="Deprecated in favour of `get_latest_review`") + @deprecated(version="2.5.3", reason="Deprecated in favour of `get_latest_review`") def get_review_status_and_comment(self): """ Return a tuple of review status and comment. """ try: # pylint: disable=no-member - status = self.reviews.latest('date_modified').status - comment = self.reviews.latest('date_modified').get_note_text() + status = self.reviews.latest("date_modified").status + comment = self.reviews.latest("date_modified").get_note_text() return status, comment except SubmissionReview.DoesNotExist: return None @@ -482,7 +525,7 @@ def get_latest_review(self): Used in favour of `get_review_status_and_comment`. """ try: - return self.reviews.latest('date_modified') + return self.reviews.latest("date_modified") except SubmissionReview.DoesNotExist: return None @@ -495,12 +538,12 @@ class Instance(models.Model, InstanceBaseClass): json = JSONField(default=dict, null=False) xml = models.TextField() user = models.ForeignKey( - User, related_name='instances', null=True, on_delete=models.SET_NULL) + User, related_name="instances", null=True, on_delete=models.SET_NULL + ) xform = models.ForeignKey( - 'logger.XForm', null=False, related_name='instances', - on_delete=models.CASCADE) - survey_type = models.ForeignKey( - 'logger.SurveyType', on_delete=models.PROTECT) + "logger.XForm", null=False, related_name="instances", on_delete=models.CASCADE + ) + survey_type = models.ForeignKey("logger.SurveyType", on_delete=models.PROTECT) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) @@ -510,8 +553,9 @@ class Instance(models.Model, InstanceBaseClass): # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) - deleted_by = models.ForeignKey(User, related_name='deleted_instances', - null=True, on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, related_name="deleted_instances", null=True, on_delete=models.SET_NULL + ) # this will be edited when we need to create a new InstanceHistory object last_edited = models.DateTimeField(null=True, default=None) @@ -521,9 +565,8 @@ class Instance(models.Model, InstanceBaseClass): # we add the following additional statuses: # - submitted_via_web # - imported_via_csv - status = models.CharField(max_length=20, - default=u'submitted_via_web') - uuid = models.CharField(max_length=249, default=u'', db_index=True) + status = models.CharField(max_length=20, default="submitted_via_web") + uuid = models.CharField(max_length=249, default="", db_index=True) version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True) # store a geographic objects associated with this instance @@ -531,25 +574,23 @@ class Instance(models.Model, InstanceBaseClass): # Keep track of whether all media attachments have been received media_all_received = models.NullBooleanField( - _("Received All Media Attachemts"), - null=True, - default=True) - total_media = models.PositiveIntegerField(_("Total Media Attachments"), - null=True, - default=0) - media_count = models.PositiveIntegerField(_("Received Media Attachments"), - null=True, - default=0) - checksum = models.CharField(max_length=64, null=True, blank=True, - db_index=True) + _("Received All Media Attachemts"), null=True, default=True + ) + total_media = models.PositiveIntegerField( + _("Total Media Attachments"), null=True, default=0 + ) + media_count = models.PositiveIntegerField( + _("Received Media Attachments"), null=True, default=0 + ) + checksum = models.CharField(max_length=64, null=True, blank=True, db_index=True) # Keep track of submission reviews, only query reviews if true has_a_review = models.BooleanField(_("has_a_review"), default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = ('xform', 'uuid') + app_label = "logger" + unique_together = ("xform", "uuid") @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now(), user=None): @@ -582,21 +623,22 @@ def get_expected_media(self): """ Returns a list of expected media files from the submission data. """ - if not hasattr(self, '_expected_media'): + if not hasattr(self, "_expected_media"): # pylint: disable=no-member data = self.get_dict() media_list = [] - if 'encryptedXmlFile' in data and self.xform.encrypted: - media_list.append(data['encryptedXmlFile']) - if 'media' in data: + if "encryptedXmlFile" in data and self.xform.encrypted: + media_list.append(data["encryptedXmlFile"]) + if "media" in data: # pylint: disable=no-member - media_list.extend([i['media/file'] for i in data['media']]) + media_list.extend([i["media/file"] for i in data["media"]]) else: - media_xpaths = (self.xform.get_media_survey_xpaths() + - self.xform.get_osm_survey_xpaths()) + media_xpaths = ( + self.xform.get_media_survey_xpaths() + + self.xform.get_osm_survey_xpaths() + ) for media_xpath in media_xpaths: - media_list.extend( - get_values_matching_key(data, media_xpath)) + media_list.extend(get_values_matching_key(data, media_xpath)) # pylint: disable=attribute-defined-outside-init self._expected_media = list(set(media_list)) @@ -607,7 +649,7 @@ def num_of_media(self): """ Returns number of media attachments expected in the submission. """ - if not hasattr(self, '_num_of_media'): + if not hasattr(self, "_num_of_media"): # pylint: disable=attribute-defined-outside-init self._num_of_media = len(self.get_expected_media()) @@ -615,15 +657,18 @@ def num_of_media(self): @property def attachments_count(self): - return self.attachments.filter( - name__in=self.get_expected_media() - ).distinct('name').order_by('name').count() + return ( + self.attachments.filter(name__in=self.get_expected_media()) + .distinct("name") + .order_by("name") + .count() + ) def save(self, *args, **kwargs): - force = kwargs.get('force') + force = kwargs.get("force") if force: - del kwargs['force'] + del kwargs["force"] self._check_is_merged_dataset() self._check_active(force) @@ -650,19 +695,18 @@ def soft_delete_attachments(self, user=None): """ Soft deletes an attachment by adding a deleted_at timestamp. """ - queryset = self.attachments.filter( - ~Q(name__in=self.get_expected_media())) - kwargs = {'deleted_at': timezone.now()} + queryset = self.attachments.filter(~Q(name__in=self.get_expected_media())) + kwargs = {"deleted_at": timezone.now()} if user: - kwargs.update({'deleted_by': user}) + kwargs.update({"deleted_by": user}) queryset.update(**kwargs) def post_save_submission(sender, instance=None, created=False, **kwargs): if instance.deleted_at is not None: - _update_submission_count_for_today(instance.xform_id, - incr=False, - date_created=instance.date_created) + _update_submission_count_for_today( + instance.xform_id, incr=False, date_created=instance.date_created + ) if ASYNC_POST_SUBMISSION_PROCESSING_ENABLED: update_xform_submission_count.apply_async(args=[instance.pk, created]) @@ -675,25 +719,29 @@ def post_save_submission(sender, instance=None, created=False, **kwargs): update_project_date_modified(instance.pk, created) -post_save.connect(post_save_submission, sender=Instance, - dispatch_uid='post_save_submission') +post_save.connect( + post_save_submission, sender=Instance, dispatch_uid="post_save_submission" +) -post_delete.connect(update_xform_submission_count_delete, sender=Instance, - dispatch_uid='update_xform_submission_count_delete') +post_delete.connect( + update_xform_submission_count_delete, + sender=Instance, + dispatch_uid="update_xform_submission_count_delete", +) class InstanceHistory(models.Model, InstanceBaseClass): - class Meta: - app_label = 'logger' + app_label = "logger" xform_instance = models.ForeignKey( - Instance, related_name='submission_history', on_delete=models.CASCADE) + Instance, related_name="submission_history", on_delete=models.CASCADE + ) user = models.ForeignKey(User, null=True, on_delete=models.CASCADE) xml = models.TextField() # old instance id - uuid = models.CharField(max_length=249, default=u'') + uuid = models.CharField(max_length=249, default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -759,9 +807,7 @@ def media_all_received(self): def _set_parser(self): if not hasattr(self, "_parser"): - self._parser = XFormInstanceParser( - self.xml, self.xform_instance.xform - ) + self._parser = XFormInstanceParser(self.xml, self.xform_instance.xform) @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): diff --git a/onadata/apps/logger/models/widget.py b/onadata/apps/logger/models/widget.py index cfb48fb7ff..67b447ecc1 100644 --- a/onadata/apps/logger/models/widget.py +++ b/onadata/apps/logger/models/widget.py @@ -1,5 +1,4 @@ from builtins import str as text -from past.builtins import basestring from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType @@ -12,50 +11,46 @@ from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm -from onadata.libs.utils.chart_tools import (DATA_TYPE_MAP, - _flatten_multiple_dict_into_one, - _use_labels_from_group_by_name, - get_field_choices, - get_field_from_field_xpath, - get_field_label) -from onadata.libs.utils.common_tags import (NUMERIC_LIST, SELECT_ONE, - SUBMISSION_TIME) +from onadata.libs.utils.chart_tools import ( + DATA_TYPE_MAP, + _flatten_multiple_dict_into_one, + _use_labels_from_group_by_name, + get_field_choices, + get_field_from_field_xpath, + get_field_label, +) +from onadata.libs.utils.common_tags import NUMERIC_LIST, SELECT_ONE, SUBMISSION_TIME from onadata.libs.utils.common_tools import get_uuid class Widget(OrderedModel): - CHARTS = 'charts' + CHARTS = "charts" # Other widgets types to be added later - WIDGETS_TYPES = ((CHARTS, 'Charts'), ) + WIDGETS_TYPES = ((CHARTS, "Charts"),) # Will hold either XForm or DataView Model content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") - widget_type = models.CharField( - max_length=25, choices=WIDGETS_TYPES, default=CHARTS) + widget_type = models.CharField(max_length=25, choices=WIDGETS_TYPES, default=CHARTS) view_type = models.CharField(max_length=50) column = models.CharField(max_length=255) - group_by = models.CharField( - null=True, default=None, max_length=255, blank=True) - - title = models.CharField( - null=True, default=None, max_length=255, blank=True) - description = models.CharField( - null=True, default=None, max_length=255, blank=True) - aggregation = models.CharField( - null=True, default=None, max_length=255, blank=True) + group_by = models.CharField(null=True, default=None, max_length=255, blank=True) + + title = models.CharField(null=True, default=None, max_length=255, blank=True) + description = models.CharField(null=True, default=None, max_length=255, blank=True) + aggregation = models.CharField(null=True, default=None, max_length=255, blank=True) key = models.CharField(db_index=True, unique=True, max_length=32) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - order_with_respect_to = 'content_type' + order_with_respect_to = "content_type" metadata = JSONField(default=dict, blank=True) class Meta(OrderedModel.Meta): - app_label = 'logger' + app_label = "logger" def save(self, *args, **kwargs): @@ -77,61 +72,60 @@ def query_data(cls, widget): field = get_field_from_field_xpath(column, xform) - if isinstance(field, basestring) and field == SUBMISSION_TIME: - field_label = 'Submission Time' - field_xpath = '_submission_time' - field_type = 'datetime' - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') + if isinstance(field, str) and field == SUBMISSION_TIME: + field_label = "Submission Time" + field_xpath = "_submission_time" + field_type = "datetime" + data_type = DATA_TYPE_MAP.get(field_type, "categorized") else: field_type = field.type - data_type = DATA_TYPE_MAP.get(field.type, 'categorized') + data_type = DATA_TYPE_MAP.get(field.type, "categorized") field_xpath = field.get_abbreviated_xpath() field_label = get_field_label(field) columns = [ - SimpleField( - field="json->>'%s'" % text(column), - alias='{}'.format(column)), - CountField( - field="json->>'%s'" % text(column), - alias='count') + SimpleField(field="json->>'%s'" % text(column), alias="{}".format(column)), + CountField(field="json->>'%s'" % text(column), alias="count"), ] if group_by: if field_type in NUMERIC_LIST: column_field = SimpleField( - field="json->>'%s'" % text(column), - cast="float", - alias=column) + field="json->>'%s'" % text(column), cast="float", alias=column + ) else: column_field = SimpleField( - field="json->>'%s'" % text(column), alias=column) + field="json->>'%s'" % text(column), alias=column + ) # build inner query - inner_query_columns = \ - [column_field, - SimpleField(field="json->>'%s'" % text(group_by), - alias=group_by), - SimpleField(field="xform_id"), - SimpleField(field="deleted_at")] + inner_query_columns = [ + column_field, + SimpleField(field="json->>'%s'" % text(group_by), alias=group_by), + SimpleField(field="xform_id"), + SimpleField(field="deleted_at"), + ] inner_query = Query().from_table(Instance, inner_query_columns) # build group-by query if field_type in NUMERIC_LIST: columns = [ - SimpleField(field=group_by, alias='%s' % group_by), + SimpleField(field=group_by, alias="%s" % group_by), SumField(field=column, alias="sum"), - AvgField(field=column, alias="mean") + AvgField(field=column, alias="mean"), ] elif field_type == SELECT_ONE: columns = [ - SimpleField(field=column, alias='%s' % column), - SimpleField(field=group_by, alias='%s' % group_by), - CountField(field="*", alias='count') + SimpleField(field=column, alias="%s" % column), + SimpleField(field=group_by, alias="%s" % group_by), + CountField(field="*", alias="count"), ] - query = Query().from_table({'inner_query': inner_query}, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table({"inner_query": inner_query}, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) if field_type == SELECT_ONE: query.group_by(column).group_by(group_by) @@ -139,8 +133,11 @@ def query_data(cls, widget): query.group_by(group_by) else: - query = Query().from_table(Instance, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table(Instance, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) query.group_by("json->>'%s'" % text(column)) # run query @@ -148,14 +145,14 @@ def query_data(cls, widget): # flatten multiple dict if select one with group by if field_type == SELECT_ONE and group_by: - records = _flatten_multiple_dict_into_one(column, group_by, - records) + records = _flatten_multiple_dict_into_one(column, group_by, records) # use labels if group by if group_by: group_by_field = get_field_from_field_xpath(group_by, xform) choices = get_field_choices(group_by, xform) records = _use_labels_from_group_by_name( - group_by, group_by_field, data_type, records, choices=choices) + group_by, group_by_field, data_type, records, choices=choices + ) return { "field_type": field_type, @@ -163,5 +160,5 @@ def query_data(cls, widget): "field_xpath": field_xpath, "field_label": field_label, "grouped_by": group_by, - "data": records + "data": records, } diff --git a/onadata/apps/logger/tests/models/test_xform.py b/onadata/apps/logger/tests/models/test_xform.py index 7f400bee60..15a425ac0f 100644 --- a/onadata/apps/logger/tests/models/test_xform.py +++ b/onadata/apps/logger/tests/models/test_xform.py @@ -5,11 +5,9 @@ import os from builtins import str as text -from past.builtins import basestring # pylint: disable=redefined-builtin from onadata.apps.logger.models import Instance, XForm -from onadata.apps.logger.models.xform import (DuplicateUUIDError, - check_xform_uuid) +from onadata.apps.logger.models.xform import DuplicateUUIDError, check_xform_uuid from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.xform_instance_parser import XLSFormError @@ -18,6 +16,7 @@ class TestXForm(TestBase): """ Test XForm model. """ + def test_submission_count(self): """ Test submission count does not include deleted submissions. @@ -43,14 +42,17 @@ def test_set_title_unicode_error(self): """ xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../..", "fixtures", "tutorial", "tutorial_arabic_labels.xls" + "../..", + "fixtures", + "tutorial", + "tutorial_arabic_labels.xls", ) self._publish_xls_file_and_set_xform(xls_file_path) - self.assertTrue(isinstance(self.xform.xml, basestring)) + self.assertTrue(isinstance(self.xform.xml, str)) # change title - self.xform.title = u'Random Title' + self.xform.title = "Random Title" self.assertNotIn(self.xform.title, self.xform.xml) @@ -65,7 +67,7 @@ def test_version_length(self): """Test Xform.version can store more than 12 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - xform.version = u'12345678901234567890' + xform.version = "12345678901234567890" xform.save() self.assertTrue(len(xform.version) > 12) @@ -89,7 +91,7 @@ def test_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() @@ -103,7 +105,7 @@ def test_soft_delete(self): # deleted-at suffix is present self.assertIn("-deleted-at-", xform.id_string) self.assertIn("-deleted-at-", xform.sms_id_string) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_get_survey_element(self): """ @@ -127,7 +129,7 @@ def test_get_survey_element(self): | | fruity | orange | Orange | | | fruity | mango | Mango | """ - kwargs = {'name': 'favs', 'title': 'Fruits', 'id_string': 'favs'} + kwargs = {"name": "favs", "title": "Fruits", "id_string": "favs"} survey = self.md_to_pyxform_survey(markdown_xlsform, kwargs) xform = XForm() xform._survey = survey # pylint: disable=W0212 @@ -136,7 +138,7 @@ def test_get_survey_element(self): self.assertIsNone(xform.get_survey_element("non_existent")) # get fruita element by name - fruita = xform.get_survey_element('fruita') + fruita = xform.get_survey_element("fruita") self.assertEqual(fruita.get_abbreviated_xpath(), "a/fruita") # get exact choices element from choice abbreviated xpath @@ -149,7 +151,7 @@ def test_get_survey_element(self): fruitb_o = xform.get_survey_element("b/fruitb/orange") self.assertEqual(fruitb_o.get_abbreviated_xpath(), "b/fruitb/orange") - self.assertEqual(xform.get_child_elements('NoneExistent'), []) + self.assertEqual(xform.get_child_elements("NoneExistent"), []) def test_check_xform_uuid(self): """ @@ -173,8 +175,10 @@ def test_id_string_max_length_on_soft_delete(self): """ self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string" + ) xform.id_string = new_string xform.sms_id_string = new_string @@ -187,15 +191,13 @@ def test_id_string_max_length_on_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() - d_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') - d_sms_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') + d_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") + d_sms_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") # deleted_at is not None self.assertIsNotNone(xform.deleted_at) @@ -209,15 +211,17 @@ def test_id_string_max_length_on_soft_delete(self): self.assertIn(xform.sms_id_string, d_sms_id_string) self.assertEqual(xform.id_string, d_id_string[:100]) self.assertEqual(xform.sms_id_string, d_sms_id_string[:100]) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_id_string_length(self): """Test Xform.id_string cannot store more than 100 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string_" \ - "before_save" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string_" + "before_save" + ) xform.id_string = new_string xform.sms_id_string = new_string diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index ecb3f599d1..aab4dbf687 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -17,11 +17,14 @@ from django.core.validators import URLValidator from django.db import IntegrityError, models from django.db.models.signals import post_delete, post_save -from past.builtins import basestring from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE, safe_delete -from onadata.libs.utils.common_tags import (GOOGLE_SHEET_DATA_TYPE, TEXTIT, - XFORM_META_PERMS, TEXTIT_DETAILS) +from onadata.libs.utils.common_tags import ( + GOOGLE_SHEET_DATA_TYPE, + TEXTIT, + XFORM_META_PERMS, + TEXTIT_DETAILS, +) CHUNK_SIZE = 1024 INSTANCE_MODEL_NAME = "instance" @@ -46,16 +49,18 @@ def is_valid_url(uri): def upload_to(instance, filename): username = None - if instance.content_object.user is None and \ - instance.content_type.model == INSTANCE_MODEL_NAME: + if ( + instance.content_object.user is None + and instance.content_type.model == INSTANCE_MODEL_NAME + ): username = instance.content_object.xform.user.username else: username = instance.content_object.user.username - if instance.data_type == 'media': - return os.path.join(username, 'formid-media', filename) + if instance.data_type == "media": + return os.path.join(username, "formid-media", filename) - return os.path.join(username, 'docs', filename) + return os.path.join(username, "docs", filename) def save_metadata(metadata_obj): @@ -69,34 +74,35 @@ def save_metadata(metadata_obj): def get_default_content_type(): content_object, created = ContentType.objects.get_or_create( - app_label="logger", model=XFORM_MODEL_NAME) + app_label="logger", model=XFORM_MODEL_NAME + ) return content_object.id -def unique_type_for_form(content_object, - data_type, - data_value=None, - data_file=None): +def unique_type_for_form(content_object, data_type, data_value=None, data_file=None): """ Ensure that each metadata object has unique xform and data_type fields return the metadata object """ - defaults = {'data_value': data_value} if data_value else {} + defaults = {"data_value": data_value} if data_value else {} content_type = ContentType.objects.get_for_model(content_object) if data_value is None and data_file is None: result = MetaData.objects.filter( - object_id=content_object.id, content_type=content_type, - data_type=data_type).first() + object_id=content_object.id, content_type=content_type, data_type=data_type + ).first() else: result, created = MetaData.objects.update_or_create( - object_id=content_object.id, content_type=content_type, - data_type=data_type, defaults=defaults) + object_id=content_object.id, + content_type=content_type, + data_type=data_type, + defaults=defaults, + ) if data_file: - if result.data_value is None or result.data_value == '': + if result.data_value is None or result.data_value == "": result.data_value = data_file.name result.data_file = data_file result.data_file_type = data_file.content_type @@ -107,15 +113,15 @@ def unique_type_for_form(content_object, def type_for_form(content_object, data_type): content_type = ContentType.objects.get_for_model(content_object) - return MetaData.objects.filter(object_id=content_object.id, - content_type=content_type, - data_type=data_type) + return MetaData.objects.filter( + object_id=content_object.id, content_type=content_type, data_type=data_type + ) def create_media(media): """Download media link""" if is_valid_url(media.data_value): - filename = media.data_value.split('/')[-1] + filename = media.data_value.split("/")[-1] data_file = NamedTemporaryFile() content_type = mimetypes.guess_type(filename) with closing(requests.get(media.data_value, stream=True)) as r: @@ -127,8 +133,8 @@ def create_media(media): data_file.seek(os.SEEK_SET) media.data_value = filename media.data_file = InMemoryUploadedFile( - data_file, 'data_file', filename, content_type, - size, charset=None) + data_file, "data_file", filename, content_type, size, charset=None + ) return media @@ -147,7 +153,7 @@ def media_resources(media_list, download=False): """ data = [] for media in media_list: - if media.data_file.name == '' and download: + if media.data_file.name == "" and download: media = create_media(media) if media: @@ -167,17 +173,17 @@ class MetaData(models.Model): date_created = models.DateTimeField(null=True, auto_now_add=True) date_modified = models.DateTimeField(null=True, auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) - content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, - default=get_default_content_type) + content_type = models.ForeignKey( + ContentType, on_delete=models.CASCADE, default=get_default_content_type + ) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") objects = models.Manager() class Meta: - app_label = 'main' - unique_together = ('object_id', 'data_type', 'data_value', - 'content_type') + app_label = "main" + unique_together = ("object_id", "data_type", "data_value", "content_type") def save(self, *args, **kwargs): self._set_hash() @@ -185,7 +191,7 @@ def save(self, *args, **kwargs): @property def hash(self): - if self.file_hash is not None and self.file_hash != '': + if self.file_hash is not None and self.file_hash != "": return self.file_hash else: return self._set_hash() @@ -196,19 +202,19 @@ def _set_hash(self): file_exists = self.data_file.storage.exists(self.data_file.name) - if (file_exists and self.data_file.name != '') \ - or (not file_exists and self.data_file): + if (file_exists and self.data_file.name != "") or ( + not file_exists and self.data_file + ): try: self.data_file.seek(os.SEEK_SET) except IOError: - return '' + return "" else: - self.file_hash = 'md5:%s' % md5( - self.data_file.read()).hexdigest() + self.file_hash = "md5:%s" % md5(self.data_file.read()).hexdigest() return self.file_hash - return '' + return "" def soft_delete(self): """ @@ -222,12 +228,12 @@ def soft_delete(self): @staticmethod def public_link(content_object, data_value=None): - data_type = 'public_link' + data_type = "public_link" if data_value is False: - data_value = 'False' + data_value = "False" metadata = unique_type_for_form(content_object, data_type, data_value) # make text field a boolean - return metadata and metadata.data_value == 'True' + return metadata and metadata.data_value == "True" @staticmethod def set_google_sheet_details(content_object, data_value=None): @@ -243,7 +249,7 @@ def get_google_sheet_details(obj): :param content_object_pk: xform primary key :return dictionary containing google sheet details """ - if isinstance(obj, basestring): + if isinstance(obj, str): metadata_data_value = obj else: metadata = MetaData.objects.filter( @@ -252,57 +258,55 @@ def get_google_sheet_details(obj): metadata_data_value = metadata and metadata.data_value if metadata_data_value: - data_list = metadata_data_value.split('|') + data_list = metadata_data_value.split("|") if data_list: # the data_list format is something like ['A a', 'B b c'] and # the list comprehension and dict cast results to # {'A': 'a', 'B': 'b c'} - return dict( - [tuple(a.strip().split(' ', 1)) for a in data_list]) + return dict([tuple(a.strip().split(" ", 1)) for a in data_list]) @staticmethod def published_by_formbuilder(content_object, data_value=None): - data_type = 'published_by_formbuilder' + data_type = "published_by_formbuilder" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_url(content_object, data_value=None): - data_type = 'enketo_url' + data_type = "enketo_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_preview_url(content_object, data_value=None): - data_type = 'enketo_preview_url' + data_type = "enketo_preview_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_single_submit_url(content_object, data_value=None): - data_type = 'enketo_single_submit_url' + data_type = "enketo_single_submit_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def form_license(content_object, data_value=None): - data_type = 'form_license' + data_type = "form_license" obj = unique_type_for_form(content_object, data_type, data_value) return (obj and obj.data_value) or None @staticmethod def data_license(content_object, data_value=None): - data_type = 'data_license' + data_type = "data_license" obj = unique_type_for_form(content_object, data_type, data_value) return (obj and obj.data_value) or None @staticmethod def source(content_object, data_value=None, data_file=None): - data_type = 'source' - return unique_type_for_form( - content_object, data_type, data_value, data_file) + data_type = "source" + return unique_type_for_form(content_object, data_type, data_value, data_file) @staticmethod def supporting_docs(content_object, data_file=None): - data_type = 'supporting_doc' + data_type = "supporting_doc" if data_file: content_type = ContentType.objects.get_for_model(content_object) @@ -312,24 +316,26 @@ def supporting_docs(content_object, data_file=None): object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_file.content_type - }) + "data_file": data_file, + "data_file_type": data_file.content_type, + }, + ) return type_for_form(content_object, data_type) @staticmethod def media_upload(content_object, data_file=None, download=False): - data_type = 'media' + data_type = "media" if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type in allowed_types: - content_type = ContentType.objects.get_for_model( - content_object) + content_type = ContentType.objects.get_for_model(content_object) media, created = MetaData.objects.update_or_create( data_type=data_type, @@ -337,85 +343,91 @@ def media_upload(content_object, data_file=None, download=False): object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_content_type - }) - return media_resources( - type_for_form(content_object, data_type), download) + "data_file": data_file, + "data_file_type": data_content_type, + }, + ) + return media_resources(type_for_form(content_object, data_type), download) @staticmethod def media_add_uri(content_object, uri): """Add a uri as a media resource""" - data_type = 'media' + data_type = "media" if is_valid_url(uri): media, created = MetaData.objects.update_or_create( data_type=data_type, data_value=uri, defaults={ - 'content_object': content_object, - }) + "content_object": content_object, + }, + ) @staticmethod def mapbox_layer_upload(content_object, data=None): - data_type = 'mapbox_layer' - if data and not MetaData.objects.filter(object_id=content_object.id, - data_type='mapbox_layer'): - s = '' + data_type = "mapbox_layer" + if data and not MetaData.objects.filter( + object_id=content_object.id, data_type="mapbox_layer" + ): + s = "" for key in data: - s = s + data[key] + '||' + s = s + data[key] + "||" content_type = ContentType.objects.get_for_model(content_object) - mapbox_layer = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=s) + mapbox_layer = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=s, + ) mapbox_layer.save() if type_for_form(content_object, data_type): - values = type_for_form( - content_object, data_type)[0].data_value.split('||') + values = type_for_form(content_object, data_type)[0].data_value.split("||") data_values = {} - data_values['map_name'] = values[0] - data_values['link'] = values[1] - data_values['attribution'] = values[2] - data_values['id'] = type_for_form(content_object, data_type)[0].id + data_values["map_name"] = values[0] + data_values["link"] = values[1] + data_values["attribution"] = values[2] + data_values["id"] = type_for_form(content_object, data_type)[0].id return data_values else: return None @staticmethod def external_export(content_object, data_value=None): - data_type = 'external_export' + data_type = "external_export" if data_value: content_type = ContentType.objects.get_for_model(content_object) - result = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=data_value) + result = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=data_value, + ) result.save() return result return MetaData.objects.filter( - object_id=content_object.id, data_type=data_type).order_by('-id') + object_id=content_object.id, data_type=data_type + ).order_by("-id") @property def external_export_url(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") return parts[1] if len(parts) > 1 else None @property def external_export_name(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") return parts[0] if len(parts) > 1 else None @property def external_export_template(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") - return parts[1].replace('xls', 'templates') if len(parts) > 1 else None + return parts[1].replace("xls", "templates") if len(parts) > 1 else None @staticmethod def textit(content_object, data_value=None): @@ -432,34 +444,32 @@ def textit_flow_details(content_object, data_value: str = ""): @property def is_linked_dataset(self): - return ( - isinstance(self.data_value, basestring) and - (self.data_value.startswith('xform') or - self.data_value.startswith('dataview')) + return isinstance(self.data_value, str) and ( + self.data_value.startswith("xform") + or self.data_value.startswith("dataview") ) @staticmethod def xform_meta_permission(content_object, data_value=None): data_type = XFORM_META_PERMS - return unique_type_for_form( - content_object, data_type, data_value) + return unique_type_for_form(content_object, data_type, data_value) @staticmethod def submission_review(content_object, data_value=None): - data_type = 'submission_review' + data_type = "submission_review" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def instance_csv_imported_by(content_object, data_value=None): - data_type = 'imported_via_csv_by' + data_type = "imported_via_csv_by" return unique_type_for_form(content_object, data_type, data_value) def clear_cached_metadata_instance_object( - sender, instance=None, created=False, **kwargs): - safe_delete('{}{}'.format( - XFORM_METADATA_CACHE, instance.object_id)) + sender, instance=None, created=False, **kwargs +): + safe_delete("{}{}".format(XFORM_METADATA_CACHE, instance.object_id)) def update_attached_object(sender, instance=None, created=False, **kwargs): @@ -467,9 +477,16 @@ def update_attached_object(sender, instance=None, created=False, **kwargs): instance.content_object.save() -post_save.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_object') -post_save.connect(update_attached_object, sender=MetaData, - dispatch_uid='update_attached_xform') -post_delete.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_delete') +post_save.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_object", +) +post_save.connect( + update_attached_object, sender=MetaData, dispatch_uid="update_attached_xform" +) +post_delete.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_delete", +) diff --git a/onadata/apps/main/tests/test_form_enter_data.py b/onadata/apps/main/tests/test_form_enter_data.py index b27dbc51f4..27b1ebca23 100644 --- a/onadata/apps/main/tests/test_form_enter_data.py +++ b/onadata/apps/main/tests/test_form_enter_data.py @@ -10,7 +10,6 @@ from future.moves.urllib.parse import urlparse from httmock import HTTMock, urlmatch from nose import SkipTest -from past.builtins import basestring from onadata.apps.logger.views import enter_data from onadata.apps.main.models import MetaData @@ -39,9 +38,7 @@ def enketo_mock_http(url, request): def enketo_error_mock(url, request): response = requests.Response() response.status_code = 400 - response._content = ( - '{"message": "no account exists for this OpenRosa server"}' - ) + response._content = '{"message": "no account exists for this OpenRosa server"}' return response @@ -80,8 +77,8 @@ def test_enketo_remote_server(self): server_url = "https://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def test_enketo_url_with_http_protocol_on_formlist(self): if not self._running_enketo(): @@ -90,9 +87,9 @@ def test_enketo_url_with_http_protocol_on_formlist(self): server_url = "http://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIn("http:", url['url']) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIn("http:", url["url"]) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def _get_grcode_view_response(self): factory = RequestFactory() @@ -105,9 +102,7 @@ def _get_grcode_view_response(self): def test_qrcode_view(self): with HTTMock(enketo_mock): response = self._get_grcode_view_response() - self.assertContains( - response, "data:image/png;base64,", status_code=200 - ) + self.assertContains(response, "data:image/png;base64,", status_code=200) def test_qrcode_view_with_enketo_error(self): with HTTMock(enketo_error_mock): @@ -121,9 +116,7 @@ def test_enter_data_redir(self): factory = RequestFactory() request = factory.get("/") request.user = self.user - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) # make sure response redirect to an enketo site enketo_base_url = urlparse(settings.ENKETO_URL).netloc redirected_base_url = urlparse(response["Location"]).netloc @@ -157,9 +150,7 @@ def test_public_with_link_to_share_toggle_on(self): factory = RequestFactory() request = factory.get("/") request.user = AnonymousUser() - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) self.assertEqual(response.status_code, 302) def test_enter_data_non_existent_user(self): diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index f6af0471f8..cd3c1a58d5 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -11,7 +11,6 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.http import HttpRequest from django.utils.translation import ugettext as _ -from past.builtins import basestring from onadata.apps.logger.models import XForm from onadata.apps.logger.models.instance import FormInactiveError @@ -22,42 +21,49 @@ from onadata.libs.utils.log import audit_log from onadata.libs.utils.logger_tools import create_instance -SMS_API_ERROR = 'SMS_API_ERROR' -SMS_PARSING_ERROR = 'SMS_PARSING_ERROR' -SMS_SUBMISSION_ACCEPTED = 'SMS_SUBMISSION_ACCEPTED' -SMS_SUBMISSION_REFUSED = 'SMS_SUBMISSION_REFUSED' -SMS_INTERNAL_ERROR = 'SMS_INTERNAL_ERROR' +SMS_API_ERROR = "SMS_API_ERROR" +SMS_PARSING_ERROR = "SMS_PARSING_ERROR" +SMS_SUBMISSION_ACCEPTED = "SMS_SUBMISSION_ACCEPTED" +SMS_SUBMISSION_REFUSED = "SMS_SUBMISSION_REFUSED" +SMS_INTERNAL_ERROR = "SMS_INTERNAL_ERROR" -BASE64_ALPHABET = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' - 'abcdefghijklmnopqrstuvwxyz0123456789+/=') -DEFAULT_SEPARATOR = '+' +BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz0123456789+/=" +DEFAULT_SEPARATOR = "+" DEFAULT_ALLOW_MEDIA = False -NA_VALUE = 'n/a' +NA_VALUE = "n/a" BASE64_ALPHABET = None -META_FIELDS = ('start', 'end', 'today', 'deviceid', 'subscriberid', - 'imei', 'phonenumber') -MEDIA_TYPES = ('audio', 'video', 'photo') -DEFAULT_DATE_FORMAT = '%Y-%m-%d' -DEFAULT_DATETIME_FORMAT = '%Y-%m-%d-%H:%M' -SENSITIVE_FIELDS = ('text', 'select all that apply', 'geopoint', 'barcode') +META_FIELDS = ( + "start", + "end", + "today", + "deviceid", + "subscriberid", + "imei", + "phonenumber", +) +MEDIA_TYPES = ("audio", "video", "photo") +DEFAULT_DATE_FORMAT = "%Y-%m-%d" +DEFAULT_DATETIME_FORMAT = "%Y-%m-%d-%H:%M" +SENSITIVE_FIELDS = ("text", "select all that apply", "geopoint", "barcode") def is_last(index, items): - return index == len(items) - 1 or (items[-1].get('type') == 'note' and - index == len(items) - 2) + return index == len(items) - 1 or ( + items[-1].get("type") == "note" and index == len(items) - 2 + ) def get_sms_instance_id(instance): - """ Human-friendly unique ID of a submission for latter ref/update + """Human-friendly unique ID of a submission for latter ref/update - For now, we strip down to the first 8 chars of the UUID. - Until we figure out what we really want (might as well be used - by formhub XML) """ + For now, we strip down to the first 8 chars of the UUID. + Until we figure out what we really want (might as well be used + by formhub XML)""" return instance.uuid[:8] def sms_media_to_file(file_object, name): - if isinstance(file_object, basestring): + if isinstance(file_object, str): file_object = io.BytesIO(file_object) def getsize(f): @@ -70,90 +76,92 @@ def getsize(f): name = name.strip() content_type, charset = mimetypes.guess_type(name) size = getsize(file_object) - return InMemoryUploadedFile(file=file_object, name=name, - field_name=None, content_type=content_type, - charset=charset, size=size) + return InMemoryUploadedFile( + file=file_object, + name=name, + field_name=None, + content_type=content_type, + charset=charset, + size=size, + ) def generate_instance(username, xml_file, media_files, uuid=None): - ''' Process an XForm submission as if done via HTTP + """Process an XForm submission as if done via HTTP - :param IO xml_file: file-like object containing XML XForm - :param string username: username of the Form's owner - :param list media_files: a list of UploadedFile objects - :param string uuid: an optionnal uuid for the instance. + :param IO xml_file: file-like object containing XML XForm + :param string username: username of the Form's owner + :param list media_files: a list of UploadedFile objects + :param string uuid: an optionnal uuid for the instance. - :returns a (status, message) tuple. ''' + :returns a (status, message) tuple.""" try: - instance = create_instance( - username, - xml_file, - media_files, - uuid=uuid - ) + instance = create_instance(username, xml_file, media_files, uuid=uuid) except InstanceInvalidUserError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Username or ID required.")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Username or ID required.")} except InstanceEmptyError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Received empty submission. " - u"No instance was created")} + return { + "code": SMS_INTERNAL_ERROR, + "text": _("Received empty submission. " "No instance was created"), + } except FormInactiveError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form is not active")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Form is not active")} except XForm.DoesNotExist: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form does not exist on this account")} + return { + "code": SMS_SUBMISSION_REFUSED, + "text": _("Form does not exist on this account"), + } except ExpatError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Improperly formatted XML.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Improperly formatted XML.")} except DuplicateInstance: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Duplicate submission")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Duplicate submission")} if instance is None: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Unable to create submission.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Unable to create submission.")} user = User.objects.get(username=username) - audit = { - "xform": instance.xform.id_string - } - audit_log(Actions.SUBMISSION_CREATED, - user, instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, HttpRequest()) + audit = {"xform": instance.xform.id_string} + audit_log( + Actions.SUBMISSION_CREATED, + user, + instance.xform.user, + _("Created submission on form %(id_string)s.") + % {"id_string": instance.xform.id_string}, + audit, + HttpRequest(), + ) xml_file.close() if len(media_files): [_file.close() for _file in media_files] - return {'code': SMS_SUBMISSION_ACCEPTED, - 'text': _(u"[SUCCESS] Your submission has been accepted."), - 'id': get_sms_instance_id(instance)} + return { + "code": SMS_SUBMISSION_ACCEPTED, + "text": _("[SUCCESS] Your submission has been accepted."), + "id": get_sms_instance_id(instance), + } def is_sms_related(json_survey): - ''' Whether a form is considered to want sms Support + """Whether a form is considered to want sms Support - return True if one sms-related field is defined. ''' + return True if one sms-related field is defined.""" def treat(value, key=None): if key is None: return False - if key in ('sms_field', 'sms_option') and value: - if not value.lower() in ('no', 'false'): + if key in ("sms_field", "sms_option") and value: + if not value.lower() in ("no", "false"): return True def walk(dl): if not isinstance(dl, (dict, list)): return False - iterator = [(None, e) for e in dl] \ - if isinstance(dl, list) else dl.items() + iterator = [(None, e) for e in dl] if isinstance(dl, list) else dl.items() for k, v in iterator: - if k == 'parent': + if k == "parent": continue if treat(v, k): return True @@ -165,86 +173,100 @@ def walk(dl): def check_form_sms_compatibility(form, json_survey=None): - ''' Tests all SMS related rules on the XForm representation + """Tests all SMS related rules on the XForm representation - Returns a view-compatible dict(type, text) with warnings or - a success message ''' + Returns a view-compatible dict(type, text) with warnings or + a success message""" if json_survey is None: - json_survey = form.get('form_o', {}) + json_survey = form.get("form_o", {}) def prep_return(msg, comp=None): from django.urls import reverse - error = 'alert-info' - warning = 'alert-info' - success = 'alert-success' - outro = (u"
Please check the " - u"SMS Syntax Page." % {'syntax_url': reverse('syntax')}) + error = "alert-info" + warning = "alert-info" + success = "alert-success" + outro = ( + '
Please check the ' + "SMS Syntax Page." % {"syntax_url": reverse("syntax")} + ) # no compatibility at all if not comp: alert = error - msg = (u"%(prefix)s %(msg)s" - % {'prefix': u"Your Form is not SMS-compatible" - u". If you want to later enable " - u"SMS Support, please fix:
", - 'msg': msg}) + msg = "%(prefix)s %(msg)s" % { + "prefix": "Your Form is not SMS-compatible" + ". If you want to later enable " + "SMS Support, please fix:
", + "msg": msg, + } # no blocker but could be improved elif comp == 1: alert = warning - msg = (u"%(prefix)s " - % {'prefix': u"Your form can be used with SMS, " - u"knowing that:", 'msg': msg}) + msg = "%(prefix)s " % { + "prefix": "Your form can be used with SMS, " "knowing that:", + "msg": msg, + } # SMS compatible else: - outro = u"" + outro = "" alert = success - return {'type': alert, - 'text': u"%(msg)s%(outro)s" - % {'msg': msg, 'outro': outro}} + return { + "type": alert, + "text": "%(msg)s%(outro)s" % {"msg": msg, "outro": outro}, + } # first level children. should be groups - groups = json_survey.get('children', [{}]) + groups = json_survey.get("children", [{}]) # BLOCKERS # overload SENSITIVE_FIELDS if date or datetime format contain spaces. sensitive_fields = copy.copy(SENSITIVE_FIELDS) - date_format = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - datetime_format = json_survey.get('sms_datetime_format', - DEFAULT_DATETIME_FORMAT) \ + date_format = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + datetime_format = ( + json_survey.get("sms_datetime_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) if len(date_format.split()) > 1: - sensitive_fields += ('date', ) + sensitive_fields += ("date",) if len(datetime_format.split()) > 1: - sensitive_fields += ('datetime', ) + sensitive_fields += ("datetime",) # must not contain out-of-group questions - if sum([1 for e in groups if e.get('type') != 'group']): - return prep_return(_(u"All your questions must be in groups.")) + if sum([1 for e in groups if e.get("type") != "group"]): + return prep_return(_("All your questions must be in groups.")) # all groups must have an sms_field - bad_groups = [e.get('name') for e in groups - if not e.get('sms_field', '') and - not e.get('name', '') == 'meta'] + bad_groups = [ + e.get("name") + for e in groups + if not e.get("sms_field", "") and not e.get("name", "") == "meta" + ] if len(bad_groups): - return prep_return(_(u"All your groups must have an 'sms_field' " - u"(use 'meta' prefixed ones for non-fillable " - u"groups). %s" % bad_groups[-1])) + return prep_return( + _( + "All your groups must have an 'sms_field' " + "(use 'meta' prefixed ones for non-fillable " + "groups). %s" % bad_groups[-1] + ) + ) # all select_one or select_multiple fields muts have sms_option for each. for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) for field in fields: - xlsf_type = field.get('type') - xlsf_name = field.get('name') - xlsf_choices = field.get('children') - if xlsf_type in ('select one', 'select all that apply'): + xlsf_type = field.get("type") + xlsf_name = field.get("name") + xlsf_choices = field.get("children") + if xlsf_type in ("select one", "select all that apply"): nb_choices = len(xlsf_choices) - options = list(set([c.get('sms_option', '') or None - for c in xlsf_choices])) + options = list( + set([c.get("sms_option", "") or None for c in xlsf_choices]) + ) try: options.remove(None) except ValueError: @@ -252,75 +274,93 @@ def prep_return(msg, comp=None): nb_options = len(options) if nb_choices != nb_options: return prep_return( - _(u"Not all options in the choices list for " - u"%s have an " - u"sms_option value.") % xlsf_name + _( + "Not all options in the choices list for " + "%s have an " + "sms_option value." + ) + % xlsf_name ) # has sensitive (space containing) fields in non-last position for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) last_pos = len(fields) - 1 # consider last field to be last before note if there's a trailing note - if fields[last_pos].get('type') == 'note': + if fields[last_pos].get("type") == "note": if len(fields) - 1: last_pos -= 1 for idx, field in enumerate(fields): - if idx != last_pos and field.get('type', '') in sensitive_fields: - return prep_return(_(u"Questions for which values can contain " - u"spaces are only allowed on last " - u"position of group (%s)" - % field.get('name'))) + if idx != last_pos and field.get("type", "") in sensitive_fields: + return prep_return( + _( + "Questions for which values can contain " + "spaces are only allowed on last " + "position of group (%s)" % field.get("name") + ) + ) # separator is not set or is within BASE64 alphabet and sms_allow_media - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR - sms_allow_media = bool(json_survey.get('sms_allow_media', - DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA) + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR + sms_allow_media = bool( + json_survey.get("sms_allow_media", DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA + ) if sms_allow_media and separator in BASE64_ALPHABET: - return prep_return(_(u"When allowing medias ('sms_allow_media'), your " - u"separator (%s) must be outside Base64 alphabet " - u"(letters, digits and +/=). " - u"You case use '#' instead." % separator)) + return prep_return( + _( + "When allowing medias ('sms_allow_media'), your " + "separator (%s) must be outside Base64 alphabet " + "(letters, digits and +/=). " + "You case use '#' instead." % separator + ) + ) # WARNINGS warnings = [] # sms_separator not set - if not json_survey.get('sms_separator', ''): - warnings.append(u"
  • You have not set a separator. Default '+' " - u"separator will be used.
    • ") + if not json_survey.get("sms_separator", ""): + warnings.append( + "
  • You have not set a separator. Default '+' " + "separator will be used.
    • " + ) # has date field with no sms_date_format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'date']): - warnings.append(u"
  • You have 'date' fields without " - u"explicitly setting a date format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATE_FORMAT) + if sum([1 for e in group.get("children", [{}]) if e.get("type") == "date"]): + warnings.append( + "
  • You have 'date' fields without " + "explicitly setting a date format. " + "Default (%s) will be used.
    • " % DEFAULT_DATE_FORMAT + ) break # has datetime field with no datetime format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'datetime']): - warnings.append(u"
  • You have 'datetime' fields without " - u"explicitly setting a datetime format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATETIME_FORMAT) + if sum( + [1 for e in group.get("children", [{}]) if e.get("type") == "datetime"] + ): + warnings.append( + "
  • You have 'datetime' fields without " + "explicitly setting a datetime format. " + "Default (%s) will be used.
    • " % DEFAULT_DATETIME_FORMAT + ) break # date or datetime format contain space - if 'date' in sensitive_fields: - warnings.append(u"
  • 'sms_date_format' contains space which will " - u"require 'date' questions to be positioned at " - u"the end of groups (%s).
    • " % date_format) - if 'datetime' in sensitive_fields: - warnings.append(u"
  • 'sms_datetime_format' contains space which will " - u"require 'datetime' questions to be positioned at " - u"the end of groups (%s).
    • " % datetime_format) + if "date" in sensitive_fields: + warnings.append( + "
  • 'sms_date_format' contains space which will " + "require 'date' questions to be positioned at " + "the end of groups (%s).
    • " % date_format + ) + if "datetime" in sensitive_fields: + warnings.append( + "
  • 'sms_datetime_format' contains space which will " + "require 'datetime' questions to be positioned at " + "the end of groups (%s).
    • " % datetime_format + ) if len(warnings): - return prep_return(u"".join(warnings), comp=1) + return prep_return("".join(warnings), comp=1) # Good to go - return prep_return(_(u"Note that your form is also SMS comptatible."), 2) + return prep_return(_("Note that your form is also SMS comptatible."), 2) diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 63992c77aa..3193b1a85c 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,17 +1,15 @@ -from past.builtins import basestring - import django.dispatch from onadata.apps.logger.models import XForm -xform_tags_add = django.dispatch.Signal(providing_args=['xform', 'tags']) -xform_tags_delete = django.dispatch.Signal(providing_args=['xform', 'tag']) +xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) +xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) @django.dispatch.receiver(xform_tags_add, sender=XForm) def add_tags_to_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tags = kwargs.get('tags', None) + xform = kwargs.get("xform", None) + tags = kwargs.get("tags", None) if isinstance(xform, XForm) and isinstance(tags, list): # update existing instances with the new tag for instance in xform.instances.all(): @@ -24,9 +22,9 @@ def add_tags_to_xform_instances(sender, **kwargs): @django.dispatch.receiver(xform_tags_delete, sender=XForm) def delete_tag_from_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tag = kwargs.get('tag', None) - if isinstance(xform, XForm) and isinstance(tag, basestring): + xform = kwargs.get("xform", None) + tag = kwargs.get("tag", None) + if isinstance(xform, XForm) and isinstance(tag, str): # update existing instances with the new tag for instance in xform.instances.all(): if tag in instance.tags.names(): diff --git a/onadata/libs/serializers/chart_serializer.py b/onadata/libs/serializers/chart_serializer.py index a35ab22765..69b1f7b41d 100644 --- a/onadata/libs/serializers/chart_serializer.py +++ b/onadata/libs/serializers/chart_serializer.py @@ -1,5 +1,3 @@ -from past.builtins import basestring - from django.http import Http404 from rest_framework import serializers @@ -11,37 +9,36 @@ class ChartSerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( - view_name='chart-detail', lookup_field='pk') + view_name="chart-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") class FieldsChartSerializer(serializers.ModelSerializer): - class Meta: model = XForm def to_representation(self, obj): data = {} - request = self.context.get('request') + request = self.context.get("request") if obj is not None: fields = obj.survey_elements if request: - selected_fields = request.query_params.get('fields') + selected_fields = request.query_params.get("fields") - if isinstance(selected_fields, basestring) \ - and selected_fields != 'all': - fields = selected_fields.split(',') - fields = [e for e in obj.survey_elements - if e.name in fields] + if isinstance(selected_fields, str) and selected_fields != "all": + fields = selected_fields.split(",") + fields = [e for e in obj.survey_elements if e.name in fields] if len(fields) == 0: raise Http404( - "Field %s does not not exist on the form" % fields) + "Field %s does not not exist on the form" % fields + ) for field in fields: if field.name == INSTANCE_ID: diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index 72b9253d1c..75f54d3f5e 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -1,18 +1,16 @@ import json from builtins import str as text -from past.builtins import basestring from rest_framework import serializers class JsonField(serializers.Field): - def to_representation(self, value): - if isinstance(value, basestring): + if isinstance(value, str): return json.loads(value) return value def to_internal_value(self, value): - if isinstance(value, basestring): + if isinstance(value, str): try: return json.loads(value) except ValueError as e: @@ -22,6 +20,6 @@ def to_internal_value(self, value): @classmethod def to_json(cls, data): - if isinstance(data, basestring): + if isinstance(data, str): return json.loads(data) return data diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index f0c4c710ba..6c0a78f837 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -2,7 +2,6 @@ """ Organization Serializer """ -from past.builtins import basestring # pylint: disable=redefined-builtin from django.contrib.auth.models import User from django.db.models.query import QuerySet @@ -12,9 +11,11 @@ from onadata.apps.api import tools from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (_get_first_last_names, - get_organization_members, - get_organization_owners) +from onadata.apps.api.tools import ( + _get_first_last_names, + get_organization_members, + get_organization_owners, +) from onadata.apps.main.forms import RegistrationFormUserProfile from onadata.libs.permissions import get_role_in_org from onadata.libs.serializers.fields.json_field import JsonField @@ -24,64 +25,64 @@ class OrganizationSerializer(serializers.HyperlinkedModelSerializer): """ Organization profile serializer """ + url = serializers.HyperlinkedIdentityField( - view_name='organizationprofile-detail', lookup_field='user') - org = serializers.CharField(source='user.username', max_length=30) + view_name="organizationprofile-detail", lookup_field="user" + ) + org = serializers.CharField(source="user.username", max_length=30) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) creator = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) users = serializers.SerializerMethodField() metadata = JsonField(required=False) name = serializers.CharField(max_length=30) class Meta: model = OrganizationProfile - exclude = ('created_by', 'is_organization', 'organization') - owner_only_fields = ('metadata', ) + exclude = ("created_by", "is_organization", "organization") + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): super(OrganizationSerializer, self).__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") is_permitted = ( - request and request.user and - request.user.has_perm('api.view_organizationprofile', - self.instance)) - if isinstance(self.instance, QuerySet) or not is_permitted or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + request + and request.user + and request.user.has_perm("api.view_organizationprofile", self.instance) + ) + if isinstance(self.instance, QuerySet) or not is_permitted or not request: + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def update(self, instance, validated_data): # update the user model - if 'name' in validated_data: - first_name, last_name = \ - _get_first_last_names(validated_data.get('name')) + if "name" in validated_data: + first_name, last_name = _get_first_last_names(validated_data.get("name")) instance.user.first_name = first_name instance.user.last_name = last_name instance.user.save() - return super(OrganizationSerializer, self).update( - instance, validated_data - ) + return super(OrganizationSerializer, self).update(instance, validated_data) def create(self, validated_data): - org = validated_data.get('user') + org = validated_data.get("user") if org: - org = org.get('username') + org = org.get("username") - org_name = validated_data.get('name', None) + org_name = validated_data.get("name", None) creator = None - if 'request' in self.context: - creator = self.context['request'].user + if "request" in self.context: + creator = self.context["request"].user - validated_data['organization'] = org_name + validated_data["organization"] = org_name - profile = tools.create_organization_object(org, creator, - validated_data) + profile = tools.create_organization_object(org, creator, validated_data) profile.save() return profile @@ -90,45 +91,48 @@ def validate_org(self, value): # pylint: disable=no-self-use """ Validate organization name. """ - org = value.lower() if isinstance(value, basestring) else value + org = value.lower() if isinstance(value, str) else value if org in RegistrationFormUserProfile.RESERVED_USERNAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % org - )) + raise serializers.ValidationError( + _("%s is a reserved name, please choose another" % org) + ) elif not RegistrationFormUserProfile.legal_usernames_re.search(org): - raise serializers.ValidationError(_( - u"Organization may only contain alpha-numeric characters and " - u"underscores" - )) + raise serializers.ValidationError( + _( + "Organization may only contain alpha-numeric characters and " + "underscores" + ) + ) try: User.objects.get(username=org) except User.DoesNotExist: return org - raise serializers.ValidationError(_( - u"Organization %s already exists." % org - )) + raise serializers.ValidationError(_("Organization %s already exists." % org)) def get_users(self, obj): # pylint: disable=no-self-use """ Return organization members. """ + def create_user_list(user_list): - return [{ - 'user': u.username, - 'role': get_role_in_org(u, obj), - 'first_name': u.first_name, - 'last_name': u.last_name, - 'gravatar': u.profile.gravatar - } for u in user_list] + return [ + { + "user": u.username, + "role": get_role_in_org(u, obj), + "first_name": u.first_name, + "last_name": u.last_name, + "gravatar": u.profile.gravatar, + } + for u in user_list + ] members = get_organization_members(obj) if obj else [] owners = get_organization_owners(obj) if obj else [] if owners and members: - members = members.exclude( - username__in=[user.username for user in owners]) + members = members.exclude(username__in=[user.username for user in owners]) members_list = create_user_list(members) owners_list = create_user_list(owners) diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index f2686c4131..10570f9f16 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -5,7 +5,6 @@ import copy import re -from past.builtins import basestring # pylint: disable=redefined-builtin from django.conf import settings from django.contrib.auth.models import User @@ -30,9 +29,7 @@ from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.utils.cache_tools import IS_ORG from onadata.libs.utils.analytics import track_object_event -from onadata.libs.utils.email import ( - get_verification_url, get_verification_email_data -) +from onadata.libs.utils.email import get_verification_url, get_verification_email_data RESERVED_NAMES = RegistrationFormUserProfile.RESERVED_USERNAMES LEGAL_USERNAMES_REGEX = RegistrationFormUserProfile.legal_usernames_re @@ -47,59 +44,55 @@ def _get_first_last_names(name, limit=30): # imposition of 30 characters on both first_name and last_name hence # ensure we only have 30 characters for either field - return name[:limit], name[limit:limit * 2] + return name[:limit], name[limit : limit * 2] name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_registration_params(attrs): params = copy.deepcopy(attrs) - name = params.get('name', None) - user = params.pop('user', None) + name = params.get("name", None) + user = params.pop("user", None) if user: - username = user.pop('username', None) - password = user.pop('password', None) - first_name = user.pop('first_name', None) - last_name = user.pop('last_name', None) - email = user.pop('email', None) + username = user.pop("username", None) + password = user.pop("password", None) + first_name = user.pop("first_name", None) + last_name = user.pop("last_name", None) + email = user.pop("email", None) if username: - params['username'] = username + params["username"] = username if email: - params['email'] = email + params["email"] = email if password: - params.update({'password1': password, 'password2': password}) + params.update({"password1": password, "password2": password}) if first_name: - params['first_name'] = first_name + params["first_name"] = first_name - params['last_name'] = last_name or '' + params["last_name"] = last_name or "" # For backward compatibility, Users who still use only name if name: - first_name, last_name = \ - _get_first_last_names(name) - params['first_name'] = first_name - params['last_name'] = last_name + first_name, last_name = _get_first_last_names(name) + params["first_name"] = first_name + params["last_name"] = last_name return params def _send_verification_email(redirect_url, user, request): - verification_key = (user.registrationprofile - .create_new_activation_key()) - verification_url = get_verification_url( - redirect_url, request, verification_key - ) + verification_key = user.registrationprofile.create_new_activation_key() + verification_url = get_verification_url(redirect_url, request, verification_key) email_data = get_verification_email_data( user.email, user.username, verification_url, request @@ -112,47 +105,72 @@ class UserProfileSerializer(serializers.HyperlinkedModelSerializer): """ UserProfile serializer. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', lookup_field='user') + view_name="userprofile-detail", lookup_field="user" + ) is_org = serializers.SerializerMethodField() - username = serializers.CharField(source='user.username', min_length=3, - max_length=30) + username = serializers.CharField( + source="user.username", min_length=3, max_length=30 + ) name = serializers.CharField(required=False, allow_blank=True) - first_name = serializers.CharField(source='user.first_name', - required=False, allow_blank=True, - max_length=30) - last_name = serializers.CharField(source='user.last_name', - required=False, allow_blank=True, - max_length=30) - email = serializers.EmailField(source='user.email') - website = serializers.CharField(source='home_page', required=False, - allow_blank=True) + first_name = serializers.CharField( + source="user.first_name", required=False, allow_blank=True, max_length=30 + ) + last_name = serializers.CharField( + source="user.last_name", required=False, allow_blank=True, max_length=30 + ) + email = serializers.EmailField(source="user.email") + website = serializers.CharField( + source="home_page", required=False, allow_blank=True + ) twitter = serializers.CharField(required=False, allow_blank=True) gravatar = serializers.ReadOnlyField() - password = serializers.CharField(source='user.password', allow_blank=True, - required=False) + password = serializers.CharField( + source="user.password", allow_blank=True, required=False + ) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) - id = serializers.ReadOnlyField(source='user.id') # pylint: disable=C0103 - joined_on = serializers.ReadOnlyField(source='user.date_joined') + id = serializers.ReadOnlyField(source="user.id") # pylint: disable=C0103 + joined_on = serializers.ReadOnlyField(source="user.date_joined") class Meta: model = UserProfile - fields = ('id', 'is_org', 'url', 'username', 'password', 'first_name', - 'last_name', 'email', 'city', 'country', 'organization', - 'website', 'twitter', 'gravatar', 'require_auth', 'user', - 'metadata', 'joined_on', 'name') - owner_only_fields = ('metadata', ) + fields = ( + "id", + "is_org", + "url", + "username", + "password", + "first_name", + "last_name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "metadata", + "joined_on", + "name", + ) + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): super(UserProfileSerializer, self).__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') - if isinstance(self.instance, QuerySet) or \ - (request and request.user != self.instance.user) or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") + if ( + isinstance(self.instance, QuerySet) + or (request and request.user != self.instance.user) + or not request + ): + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def get_is_org(self, obj): # pylint: disable=no-self-use @@ -160,12 +178,12 @@ def get_is_org(self, obj): # pylint: disable=no-self-use Returns True if it is an organization profile. """ if obj: - is_org = cache.get('{}{}'.format(IS_ORG, obj.pk)) + is_org = cache.get("{}{}".format(IS_ORG, obj.pk)) if is_org: return is_org is_org = is_organization(obj) - cache.set('{}{}'.format(IS_ORG, obj.pk), is_org) + cache.set("{}{}".format(IS_ORG, obj.pk), is_org) return is_org def to_representation(self, instance): @@ -173,46 +191,46 @@ def to_representation(self, instance): Serialize objects -> primitives. """ ret = super(UserProfileSerializer, self).to_representation(instance) - if 'password' in ret: - del ret['password'] + if "password" in ret: + del ret["password"] - request = self.context['request'] \ - if 'request' in self.context else None + request = self.context["request"] if "request" in self.context else None - if 'email' in ret and request is None or request.user \ - and not request.user.has_perm(CAN_VIEW_PROFILE, instance): - del ret['email'] + if ( + "email" in ret + and request is None + or request.user + and not request.user.has_perm(CAN_VIEW_PROFILE, instance) + ): + del ret["email"] - if 'first_name' in ret: - ret['name'] = u' '.join([ret.get('first_name'), - ret.get('last_name', "")]) - ret['name'] = ret['name'].strip() + if "first_name" in ret: + ret["name"] = " ".join([ret.get("first_name"), ret.get("last_name", "")]) + ret["name"] = ret["name"].strip() return ret def update(self, instance, validated_data): params = validated_data password = params.get("password1") - email = params.get('email') + email = params.get("email") # Check password if email is being updated if email and not password: raise serializers.ValidationError( - _(u'Your password is required when updating your email ' - u'address.')) + _("Your password is required when updating your email " "address.") + ) if password and not instance.user.check_password(password): - raise serializers.ValidationError(_(u'Invalid password')) + raise serializers.ValidationError(_("Invalid password")) # get user instance.user.email = email or instance.user.email - instance.user.first_name = params.get('first_name', - instance.user.first_name) + instance.user.first_name = params.get("first_name", instance.user.first_name) - instance.user.last_name = params.get('last_name', - instance.user.last_name) + instance.user.last_name = params.get("last_name", instance.user.last_name) - instance.user.username = params.get('username', instance.user.username) + instance.user.username = params.get("username", instance.user.username) instance.user.save() @@ -220,8 +238,8 @@ def update(self, instance, validated_data): instance.metadata.update({"is_email_verified": False}) instance.save() - request = self.context.get('request') - redirect_url = params.get('redirect_url') + request = self.context.get("request") + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, instance.user, request) if password: @@ -231,51 +249,48 @@ def update(self, instance, validated_data): return super(UserProfileSerializer, self).update(instance, params) @track_object_event( - user_field='user', - properties={ - 'name': 'name', - 'country': 'country'}) + user_field="user", properties={"name": "name", "country": "country"} + ) def create(self, validated_data): params = validated_data - request = self.context.get('request') + request = self.context.get("request") metadata = {} site = Site.objects.get(pk=settings.SITE_ID) try: new_user = RegistrationProfile.objects.create_inactive_user( - username=params.get('username'), - password=params.get('password1'), - email=params.get('email'), + username=params.get("username"), + password=params.get("password1"), + email=params.get("email"), site=site, - send_email=settings.SEND_EMAIL_ACTIVATION_API) + send_email=settings.SEND_EMAIL_ACTIVATION_API, + ) except IntegrityError: - raise serializers.ValidationError(_( - u"User account {} already exists".format( - params.get('username')) - )) + raise serializers.ValidationError( + _("User account {} already exists".format(params.get("username"))) + ) new_user.is_active = True - new_user.first_name = params.get('first_name') - new_user.last_name = params.get('last_name') + new_user.first_name = params.get("first_name") + new_user.last_name = params.get("last_name") new_user.save() - if getattr( - settings, 'ENABLE_EMAIL_VERIFICATION', False - ): - redirect_url = params.get('redirect_url') + if getattr(settings, "ENABLE_EMAIL_VERIFICATION", False): + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, new_user, request) created_by = request.user created_by = None if created_by.is_anonymous else created_by - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() profile = UserProfile( - user=new_user, name=params.get('first_name'), + user=new_user, + name=params.get("first_name"), created_by=created_by, - city=params.get('city', u''), - country=params.get('country', u''), - organization=params.get('organization', u''), - home_page=params.get('home_page', u''), - twitter=params.get('twitter', u''), - metadata=metadata + city=params.get("city", ""), + country=params.get("country", ""), + organization=params.get("organization", ""), + home_page=params.get("home_page", ""), + twitter=params.get("twitter", ""), + metadata=metadata, ) profile.save() return profile @@ -284,28 +299,28 @@ def validate_username(self, value): """ Validate username. """ - username = value.lower() if isinstance(value, basestring) else value + username = value.lower() if isinstance(value, str) else value if username in RESERVED_NAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % username - )) + raise serializers.ValidationError( + _("%s is a reserved name, please choose another" % username) + ) elif not LEGAL_USERNAMES_REGEX.search(username): - raise serializers.ValidationError(_( - u"username may only contain alpha-numeric characters and " - u"underscores" - )) + raise serializers.ValidationError( + _( + "username may only contain alpha-numeric characters and " + "underscores" + ) + ) elif len(username) < 3: - raise serializers.ValidationError(_( - u"Username must have 3 or more characters" - )) + raise serializers.ValidationError( + _("Username must have 3 or more characters") + ) users = User.objects.filter(username=username) if self.instance: users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"%s already exists" % username - )) + raise serializers.ValidationError(_("%s already exists" % username)) return username @@ -318,9 +333,9 @@ def validate_email(self, value): users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"This email address is already in use. " - )) + raise serializers.ValidationError( + _("This email address is already in use. ") + ) return value @@ -328,22 +343,25 @@ def validate_twitter(self, value): # pylint: disable=no-self-use """ Checks if the twitter handle is valid. """ - if isinstance(value, basestring) and value: + if isinstance(value, str) and value: match = re.search(r"^[A-Za-z0-9_]{1,15}$", value) if not match: - raise serializers.ValidationError(_( - u"Invalid twitter username {}".format(value) - )) + raise serializers.ValidationError( + _("Invalid twitter username {}".format(value)) + ) return value def validate(self, attrs): params = _get_registration_params(attrs) - if not self.instance and params.get('name') is None and \ - params.get('first_name') is None: - raise serializers.ValidationError({ - 'name': _(u"Either name or first_name should be provided") - }) + if ( + not self.instance + and params.get("name") is None + and params.get("first_name") is None + ): + raise serializers.ValidationError( + {"name": _("Either name or first_name should be provided")} + ) return params @@ -352,23 +370,38 @@ class UserProfileWithTokenSerializer(serializers.HyperlinkedModelSerializer): """ User Profile Serializer that includes the users API Tokens. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', - lookup_field='user') - username = serializers.CharField(source='user.username') - email = serializers.CharField(source='user.email') - website = serializers.CharField(source='home_page', required=False) + view_name="userprofile-detail", lookup_field="user" + ) + username = serializers.CharField(source="user.username") + email = serializers.CharField(source="user.email") + website = serializers.CharField(source="home_page", required=False) gravatar = serializers.ReadOnlyField() user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) api_token = serializers.SerializerMethodField() temp_token = serializers.SerializerMethodField() class Meta: model = UserProfile - fields = ('url', 'username', 'name', 'email', 'city', - 'country', 'organization', 'website', 'twitter', 'gravatar', - 'require_auth', 'user', 'api_token', 'temp_token') + fields = ( + "url", + "username", + "name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "api_token", + "temp_token", + ) def get_api_token(self, object): # pylint: disable=R0201,W0622 """ @@ -381,7 +414,7 @@ def get_temp_token(self, object): # pylint: disable=R0201,W0622 This should return a valid temp token for this user profile. """ token, created = TempToken.objects.get_or_create(user=object.user) - check_expired = getattr(settings, 'CHECK_EXPIRED_TEMP_TOKEN', True) + check_expired = getattr(settings, "CHECK_EXPIRED_TEMP_TOKEN", True) try: if check_expired and not created and expired(token.created): diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 1ac97a761b..4c28ef33a9 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -19,285 +19,287 @@ from django.conf import settings from django.core.files.temp import NamedTemporaryFile from openpyxl import load_workbook -from past.builtins import basestring from pyxform.builder import create_survey_from_xls from savReaderWriter import SavHeaderReader, SavReader from onadata.apps.logger.import_tools import django_file from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (_encode_for_mongo, - query_data) +from onadata.apps.viewer.models.parsed_instance import _encode_for_mongo, query_data from onadata.apps.viewer.tests.export_helpers import viewer_fixture_path -from onadata.libs.utils.csv_builder import (CSVDataFrameBuilder, - get_labels_from_columns) +from onadata.libs.utils.csv_builder import CSVDataFrameBuilder, get_labels_from_columns from onadata.libs.utils.common_tags import ( - SELECT_BIND_TYPE, MULTIPLE_SELECT_TYPE, REVIEW_COMMENT, REVIEW_DATE, - REVIEW_STATUS) + SELECT_BIND_TYPE, + MULTIPLE_SELECT_TYPE, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, +) from onadata.libs.utils.export_builder import ( decode_mongo_encoded_section_names, dict_to_joined_export, ExportBuilder, - string_to_date_with_xls_validation) + string_to_date_with_xls_validation, +) from onadata.libs.utils.export_tools import get_columns_with_hxl from onadata.libs.utils.logger_tools import create_instance def _logger_fixture_path(*args): - return os.path.join(settings.PROJECT_ROOT, 'apps', 'logger', - 'tests', 'fixtures', *args) + return os.path.join( + settings.PROJECT_ROOT, "apps", "logger", "tests", "fixtures", *args + ) class TestExportBuilder(TestBase): data = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', - 'children': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", + "children": [ { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/iceLg==creams': 'vanilla chocolate', - 'children/cartoons': - [ + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/iceLg==creams": "vanilla chocolate", + "children/cartoons": [ { - 'children/cartoons/name': 'Tom & Jerry', - 'children/cartoons/why': 'Tom is silly', + "children/cartoons/name": "Tom & Jerry", + "children/cartoons/why": "Tom is silly", }, { - 'children/cartoons/name': 'Flinstones', - 'children/cartoons/why': u"I like bam bam\u0107" + "children/cartoons/name": "Flinstones", + "children/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] - }, - { - 'children/name': 'John', - 'children/age': 2, - 'children/cartoons': [] + }, + ], }, + {"children/name": "John", "children/age": 2, "children/cartoons": []}, { - 'children/name': 'Imora', - 'children/age': 3, - 'children/cartoons': - [ + "children/name": "Imora", + "children/age": 3, + "children/cartoons": [ { - 'children/cartoons/name': 'Shrek', - 'children/cartoons/why': 'He\'s so funny' + "children/cartoons/name": "Shrek", + "children/cartoons/why": "He's so funny", }, { - 'children/cartoons/name': 'Dexter\'s Lab', - 'children/cartoons/why': 'He thinks hes smart', - 'children/cartoons/characters': - [ + "children/cartoons/name": "Dexter's Lab", + "children/cartoons/why": "He thinks hes smart", + "children/cartoons/characters": [ { - 'children/cartoons/characters/name': - 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "children/cartoons/characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'children/cartoons/characters/name': - 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "children/cartoons/characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], }, { # blank data just to be sure - 'children': [] - } + "children": [] + }, ] long_survey_data = [ { - 'name': 'Abe', - 'age': 35, - 'childrens_survey_with_a_very_lo': - [ + "name": "Abe", + "age": 35, + "childrens_survey_with_a_very_lo": [ { - 'childrens_survey_with_a_very_lo/name': 'Mike', - 'childrens_survey_with_a_very_lo/age': 5, - 'childrens_survey_with_a_very_lo/fav_colors': 'red blue', - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Mike", + "childrens_survey_with_a_very_lo/age": 5, + "childrens_survey_with_a_very_lo/fav_colors": "red blue", + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Tom & Jerry', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'Tom is silly', + "childrens_survey_with_a_very_lo/cartoons/name": "Tom & Jerry", + "childrens_survey_with_a_very_lo/cartoons/why": "Tom is silly", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Flinstones', - 'childrens_survey_with_a_very_lo/cartoons/why': - u"I like bam bam\u0107" + "childrens_survey_with_a_very_lo/cartoons/name": "Flinstones", + "childrens_survey_with_a_very_lo/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] + }, + ], }, { - 'childrens_survey_with_a_very_lo/name': 'John', - 'childrens_survey_with_a_very_lo/age': 2, - 'childrens_survey_with_a_very_lo/cartoons': [] + "childrens_survey_with_a_very_lo/name": "John", + "childrens_survey_with_a_very_lo/age": 2, + "childrens_survey_with_a_very_lo/cartoons": [], }, { - 'childrens_survey_with_a_very_lo/name': 'Imora', - 'childrens_survey_with_a_very_lo/age': 3, - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Imora", + "childrens_survey_with_a_very_lo/age": 3, + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Shrek', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He\'s so funny' + "childrens_survey_with_a_very_lo/cartoons/name": "Shrek", + "childrens_survey_with_a_very_lo/cartoons/why": "He's so funny", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Dexter\'s Lab', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He thinks hes smart', - 'childrens_survey_with_a_very_lo/cartoons/characte' - 'rs': - [ + "childrens_survey_with_a_very_lo/cartoons/name": "Dexter's Lab", + "childrens_survey_with_a_very_lo/cartoons/why": "He thinks hes smart", + "childrens_survey_with_a_very_lo/cartoons/characte" + "rs": [ { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], } ] data_utf8 = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - 'childrenLg==info': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "childrenLg==info": [ { - 'childrenLg==info/nameLg==first': 'Mike', - 'childrenLg==info/age': 5, - 'childrenLg==info/fav_colors': "red's blue's", - 'childrenLg==info/ice_creams': 'vanilla chocolate', - 'childrenLg==info/cartoons': - [ + "childrenLg==info/nameLg==first": "Mike", + "childrenLg==info/age": 5, + "childrenLg==info/fav_colors": "red's blue's", + "childrenLg==info/ice_creams": "vanilla chocolate", + "childrenLg==info/cartoons": [ { - 'childrenLg==info/cartoons/name': 'Tom & Jerry', - 'childrenLg==info/cartoons/why': 'Tom is silly', + "childrenLg==info/cartoons/name": "Tom & Jerry", + "childrenLg==info/cartoons/why": "Tom is silly", }, { - 'childrenLg==info/cartoons/name': 'Flinstones', - 'childrenLg==info/cartoons/why': + "childrenLg==info/cartoons/name": "Flinstones", + "childrenLg==info/cartoons/why": # throw in a unicode character - 'I like bam bam\u0107' - } - ] + "I like bam bam\u0107", + }, + ], } - ] + ], } ] osm_data = [ { - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road:ctr:lat': '23.708174238006087', - 'osm_road:ctr:lon': '90.40946505581161', - 'osm_road:highway': 'tertiary', - 'osm_road:lanes': 2, - 'osm_road:name': 'Patuatuli Road', - 'osm_building:building': 'yes', - 'osm_building:building:levels': 4, - 'osm_building:ctr:lat': '23.707316084046038', - 'osm_building:ctr:lon': '90.40849938337506', - 'osm_building:name': 'kol', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road:ctr:lat": "23.708174238006087", + "osm_road:ctr:lon": "90.40946505581161", + "osm_road:highway": "tertiary", + "osm_road:lanes": 2, + "osm_road:name": "Patuatuli Road", + "osm_building:building": "yes", + "osm_building:building:levels": 4, + "osm_building:ctr:lat": "23.707316084046038", + "osm_building:ctr:lon": "90.40849938337506", + "osm_building:name": "kol", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", } ] def _create_childrens_survey(self, filename="childrens_survey.xls"): - survey = create_survey_from_xls(_logger_fixture_path( - filename - ), default_name=filename.split('.')[0]) + survey = create_survey_from_xls( + _logger_fixture_path(filename), default_name=filename.split(".")[0] + ) self.dd = DataDictionary() self.dd._survey = survey return survey def test_build_sections_for_multilanguage_form(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'multi_lingual_form.xls'), - default_name='multi_lingual_form') + survey = create_survey_from_xls( + _logger_fixture_path("multi_lingual_form.xls"), + default_name="multi_lingual_form", + ) # check the default langauge - self.assertEqual( - survey.to_json_dict().get('default_language'), 'English' - ) + self.assertEqual(survey.to_json_dict().get("default_language"), "English") export_builder = ExportBuilder() export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - expected_sections = [ - survey.name] + expected_sections = [survey.name] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_sections, [s["name"] for s in export_builder.sections] + ) + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - expected_element_names = \ - ['Des fruits', 'Fruits/Aucune de ces réponses', 'Fruits/Autre', - 'Fruits/Banane', 'Fruits/Mangue', 'Fruits/Poire', 'Fruits/Pomme', - "L'age", 'Le genre', 'Nom de personne interrogée', 'Villes', - 'meta/instanceID'] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + expected_element_names = [ + "Des fruits", + "Fruits/Aucune de ces réponses", + "Fruits/Autre", + "Fruits/Banane", + "Fruits/Mangue", + "Fruits/Poire", + "Fruits/Pomme", + "L'age", + "Le genre", + "Nom de personne interrogée", + "Villes", + "meta/instanceID", + ] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) # use default language when the language passed does not exist - export_builder.language = 'Kiswahili' + export_builder.language = "Kiswahili" export_builder.set_survey(survey) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) def test_build_sections_from_survey(self): survey = self._create_childrens_survey() @@ -305,59 +307,71 @@ def test_build_sections_from_survey(self): export_builder.set_survey(survey) # test that we generate the proper sections expected_sections = [ - survey.name, 'children', 'children/cartoons', - 'children/cartoons/characters'] + survey.name, + "children", + "children/cartoons", + "children/cartoons/characters", + ] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) + expected_sections, [s["name"] for s in export_builder.sections] + ) # main section should have split geolocations expected_element_names = [ - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_longitude', - 'geo/_geolocation_latitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', 'tel/tel.mobile', - 'meta/instanceID'] + "name", + "age", + "geo/geolocation", + "geo/_geolocation_longitude", + "geo/_geolocation_latitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['xpath'] for element in section['elements']] + element_names = [element["xpath"] for element in section["elements"]] # fav_colors should have its choices split - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate'] - section = export_builder.section_by_name('children') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ] + section = export_builder.section_by_name("children") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - expected_element_names = [ - 'children/cartoons/name', 'children/cartoons/why'] - section = export_builder.section_by_name('children/cartoons') - element_names = [element['xpath'] for element in section['elements']] + expected_element_names = ["children/cartoons/name", "children/cartoons/why"] + section = export_builder.section_by_name("children/cartoons") + element_names = [element["xpath"] for element in section["elements"]] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil'] - section = \ - export_builder.section_by_name('children/cartoons/characters') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + ] + section = export_builder.section_by_name("children/cartoons/characters") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() @@ -369,68 +383,70 @@ def test_zipped_csv_export_works(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.csv".format(survey.name)))) - with open(os.path.join(temp_dir, "{0}.csv".format(survey.name)), - encoding='utf-8') as csv_file: + os.path.exists(os.path.join(temp_dir, "{0}.csv".format(survey.name))) + ) + with open( + os.path.join(temp_dir, "{0}.csv".format(survey.name)), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'childrens_survey.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "childrens_survey.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.csv"))) - with open(os.path.join(temp_dir, "children.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) + with open(os.path.join(temp_dir, "children.csv"), encoding="utf-8") as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'children.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons.csv"))) - with open(os.path.join(temp_dir, "children_cartoons.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) + with open( + os.path.join(temp_dir, "children_cartoons.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'children_cartoons.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children_cartoons.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons_characters.csv"))) - with open(os.path.join(temp_dir, "children_cartoons_characters.csv"), - encoding='utf-8') as csv_file: + os.path.exists(os.path.join(temp_dir, "children_cartoons_characters.csv")) + ) + with open( + os.path.join(temp_dir, "children_cartoons_characters.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path( - 'csvs', 'children_cartoons_characters.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children_cartoons_characters.csv"), + encoding="utf-8", + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) @@ -445,7 +461,7 @@ def test_xls_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.osm_data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) @@ -455,37 +471,57 @@ def test_xls_export_with_osm_data(self): temp_xls_file.close() expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(expected_column_headers), sorted(xls_headers)) submission = [a.value for a in rows[1]] - self.assertEqual(submission[0], '1424308569120.jpg') - self.assertEqual(submission[2], '23.708174238006087') - self.assertEqual(submission[4], 'tertiary') - self.assertEqual(submission[6], 'Patuatuli Road') - self.assertEqual(submission[11], '23.707316084046038') - self.assertEqual(submission[13], 'kol') + self.assertEqual(submission[0], "1424308569120.jpg") + self.assertEqual(submission[2], "23.708174238006087") + self.assertEqual(submission[4], "tertiary") + self.assertEqual(submission[6], "Patuatuli Road") + self.assertEqual(submission[11], "23.707316084046038") + self.assertEqual(submission[13], "kol") def test_decode_mongo_encoded_section_names(self): data = { - 'main_section': [1, 2, 3, 4], - 'sectionLg==1/info': [1, 2, 3, 4], - 'sectionLg==2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "sectionLg==1/info": [1, 2, 3, 4], + "sectionLg==2/info": [1, 2, 3, 4], } result = decode_mongo_encoded_section_names(data) expected_result = { - 'main_section': [1, 2, 3, 4], - 'section.1/info': [1, 2, 3, 4], - 'section.2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "section.1/info": [1, 2, 3, 4], + "section.2/info": [1, 2, 3, 4], } self.assertEqual(result, expected_result) @@ -493,12 +529,13 @@ def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -507,34 +544,42 @@ def test_zipped_csv_export_works_with_unicode(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['children.info/name.first', - 'children.info/age', - 'children.info/fav_colors', - 'children.info/fav_colors/red\'s', - 'children.info/fav_colors/blue\'s', - 'children.info/fav_colors/pink\'s', - 'children.info/ice_creams', - 'children.info/ice_creams/vanilla', - 'children.info/ice_creams/strawberry', - 'children.info/ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "children.info/name.first", + "children.info/age", + "children.info/fav_colors", + "children.info/fav_colors/red's", + "children.info/fav_colors/blue's", + "children.info/fav_colors/pink's", + "children.info/ice_creams", + "children.info/ice_creams/vanilla", + "children.info/ice_creams/strawberry", + "children.info/ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['children.info/fav_colors/red\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/blue\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/pink\'s'], 'False') + self.assertEqual(data["children.info/fav_colors/red's"], "True") + self.assertEqual(data["children.info/fav_colors/blue's"], "True") + self.assertEqual(data["children.info/fav_colors/pink's"], "False") # check that red and blue are set to true def test_zipped_sav_export_with_date_field(self): @@ -549,12 +594,17 @@ def test_zipped_sav_export_with_date_field(self): | choices | | | list name | name | label | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expense_date": "2013-01-03", "A/gdate": "2017-06-13", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expense_date": "2013-01-03", + "A/gdate": "2017-06-13", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -563,21 +613,18 @@ def test_zipped_sav_export_with_date_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'expense_date') - self.assertEqual(rows[1][0], b'2013-01-03') - self.assertEqual(rows[0][1], b'A.gdate') - self.assertEqual(rows[1][1], b'2017-06-13') - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][0], b"expense_date") + self.assertEqual(rows[1][0], b"2013-01-03") + self.assertEqual(rows[0][1], b"A.gdate") + self.assertEqual(rows[1][1], b"2017-06-13") + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -600,13 +647,19 @@ def test_zipped_sav_export_dynamic_select_multiple(self): | | brand | a | a | | | brand | b | b | """ # noqa - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) data = [ - {"sex": "male", "text": "his", "favorite_brand": "Generic", - "name": "Davis", "brand_known": "${text} ${favorite_brand} a"}] + { + "sex": "male", + "text": "his", + "favorite_brand": "Generic", + "name": "Davis", + "brand_known": "${text} ${favorite_brand} a", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -615,31 +668,28 @@ def test_zipped_sav_export_dynamic_select_multiple(self): zip_file.close() temp_zip_file.close() - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'sex') - self.assertEqual(rows[1][0], b'male') - self.assertEqual(rows[0][1], b'text') - self.assertEqual(rows[1][1], b'his') - self.assertEqual(rows[0][2], b'favorite_brand') - self.assertEqual(rows[1][2], b'Generic') - self.assertEqual(rows[0][3], b'name') - self.assertEqual(rows[1][3], b'Davis') - self.assertEqual(rows[0][4], b'brand_known') - self.assertEqual(rows[1][4], b'his Generic a') - self.assertEqual(rows[0][5], b'brand_known.$text') + self.assertEqual(rows[0][0], b"sex") + self.assertEqual(rows[1][0], b"male") + self.assertEqual(rows[0][1], b"text") + self.assertEqual(rows[1][1], b"his") + self.assertEqual(rows[0][2], b"favorite_brand") + self.assertEqual(rows[1][2], b"Generic") + self.assertEqual(rows[0][3], b"name") + self.assertEqual(rows[1][3], b"Davis") + self.assertEqual(rows[0][4], b"brand_known") + self.assertEqual(rows[1][4], b"his Generic a") + self.assertEqual(rows[0][5], b"brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], b'brand_known.$favorite_brand') + self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], b'brand_known.a') + self.assertEqual(rows[0][7], b"brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], b'brand_known.b') + self.assertEqual(rows[0][8], b"brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) @@ -655,31 +705,27 @@ def test_zipped_sav_export_with_zero_padded_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{'expensed': '09', - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "09", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, 'exp.sav'))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, 'exp.sav'), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0].decode('utf-8'), '09') - self.assertEqual(rows[1][4].decode('utf-8'), '2016-11-21 03:43:43') + self.assertEqual(rows[1][0].decode("utf-8"), "09") + self.assertEqual(rows[1][4].decode("utf-8"), "2016-11-21 03:43:43") def test_zipped_sav_export_with_numeric_select_one_field(self): md = """ @@ -695,12 +741,17 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 0 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -709,27 +760,24 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], b'expensed') + self.assertEqual(rows[0][0], b"expensed") self.assertEqual(rows[1][0], 1) # A/q1 1 - self.assertEqual(rows[0][1], b'A.q1') + self.assertEqual(rows[0][1], b"A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") def test_zipped_sav_export_with_duplicate_field_different_groups(self): """ @@ -760,20 +808,20 @@ def test_zipped_sav_export_with_duplicate_field_different_groups(self): | | x_y | 2 | Non | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - labels = export_builder._get_sav_value_labels({'A/allaite': 'allaite'}) - self.assertEqual(labels, {'allaite': {'1': 'Oui', '2': 'Non'}}) + labels = export_builder._get_sav_value_labels({"A/allaite": "allaite"}) + self.assertEqual(labels, {"allaite": {"1": "Oui", "2": "Non"}}) repeat_group_labels = export_builder._get_sav_value_labels( - {'A/rep/allaite': 'allaite'}) - self.assertEqual(repeat_group_labels, - {'allaite': {1: 'Yes', 2: 'No'}}) + {"A/rep/allaite": "allaite"} + ) + self.assertEqual(repeat_group_labels, {"allaite": {1: "Yes", 2: "No"}}) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") try: export_builder.to_zipped_sav(temp_zip_file.name, []) @@ -813,54 +861,55 @@ def test_split_select_multiples_choices_with_randomize_param(self): | | allow_choice_duplicates | | | Yes | """ # noqa: E501 - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) @@ -880,12 +929,18 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): | | y_n | 0 | No | | | | | | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", "A/q2": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "A/q2": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -894,13 +949,10 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) @@ -932,7 +984,7 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): self.assertEqual(rows[1][5], 0) self.assertEqual(rows[0][12], b"@_submission_time") - self.assertEqual(rows[1][12], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -947,12 +999,11 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -961,13 +1012,10 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"1") @@ -975,7 +1023,7 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -990,13 +1038,14 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): | | yes_no | 2 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "2 09", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"expensed": "2 09", "_submission_time": "2016-11-21T03:43:43.000-08:00"} + ] export_builder = ExportBuilder() export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1005,21 +1054,18 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 2) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[1][2], b'09') - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][2], b"09") + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1045,15 +1091,14 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): | | allow_choice_duplicates | | | Yes | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'} - ] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1062,9 +1107,7 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 """ @@ -1075,14 +1118,14 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 | | type | name | label | | | select_one_from_file animals.csv | q1 | Favorite animal? | """ - survey = self.md_to_pyxform_survey(xform_markdown, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(xform_markdown, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1091,9 +1134,7 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) def test_zipped_sav_export_with_duplicate_column_name(self): """ @@ -1105,13 +1146,18 @@ def test_zipped_sav_export_with_duplicate_column_name(self): | | text | Sport | Which sport | | | text | sport | Which fun sports| """ - survey = self.md_to_pyxform_survey(md, {'name': 'sports'}) - data = [{"Sport": "Basketball", "sport": "Soccer", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "sports"}) + data = [ + { + "Sport": "Basketball", + "sport": "Soccer", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1120,13 +1166,12 @@ def test_zipped_sav_export_with_duplicate_column_name(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "sports.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "sports.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "sports.sav"), - returnHeader=True) as reader: + with SavReader( + os.path.join(temp_dir, "sports.sav"), returnHeader=True + ) as reader: rows = [r for r in reader] # Check that columns are present @@ -1136,12 +1181,13 @@ def test_zipped_sav_export_with_duplicate_column_name(self): self.assertIn(b"sport", [x[0:5] for x in rows[0]]) def test_xls_export_works_with_unicode(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrenss_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrenss_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true @@ -1156,20 +1202,26 @@ def test_xls_export_works_with_unicode(self): def test_xls_export_with_hxl_adds_extra_row(self): # hxl_example.xlsx contains `instance::hxl` column whose value is #age xlsform_path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "hxl_test", "hxl_example.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "hxl_test", + "hxl_example.xlsx", + ) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) export_builder = ExportBuilder() export_builder.INCLUDE_HXL = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") survey_elements = [ survey_item[1] for survey_item in survey.items() - if survey_item[0] == 'children' + if survey_item[0] == "children" ][0] columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( @@ -1177,8 +1229,8 @@ def test_xls_export_with_hxl_adds_extra_row(self): ) export_builder.to_xls_export( - temp_xls_file.name, self.data_utf8, - columns_with_hxl=columns_with_hxl) + temp_xls_file.name, self.data_utf8, columns_with_hxl=columns_with_hxl + ) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) children_sheet = wb["hxl_example"] @@ -1187,7 +1239,7 @@ def test_xls_export_with_hxl_adds_extra_row(self): # we pick the second row because the first row has xform fieldnames rows = [row for row in children_sheet.rows] hxl_row = [a.value for a in rows[1]] - self.assertIn('#age', hxl_row) + self.assertIn("#age", hxl_row) def test_export_with_image_attachments(self): """ @@ -1208,23 +1260,29 @@ def test_export_with_image_attachments(self): 1300221157303.jpg - """.format(self.xform.id_string) - - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - media_file = django_file(path=file_path, - field_name="image1", - content_type="image/jpeg") - create_instance(self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + """.format( + self.xform.id_string + ) + + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + media_file = django_file( + path=file_path, field_name="image1", content_type="image/jpeg" + ) + create_instance( + self.user.username, + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) xdata = query_data(self.xform) - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file, xdata) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file) @@ -1232,9 +1290,11 @@ def test_export_with_image_attachments(self): self.assertTrue(children_sheet) rows = [row for row in children_sheet.rows] row = [a.value for a in rows[1]] - attachment_id = xdata[0]['_attachments'][0]['id'] - attachment_filename = xdata[0]['_attachments'][0]['filename'] - attachment_url = 'http://example.com/api/v1/files/{}?filename={}'.format(attachment_id, attachment_filename) # noqa + attachment_id = xdata[0]["_attachments"][0]["id"] + attachment_filename = xdata[0]["_attachments"][0]["filename"] + attachment_url = "http://example.com/api/v1/files/{}?filename={}".format( + attachment_id, attachment_filename + ) # noqa self.assertIn(attachment_url, row) temp_xls_file.close() @@ -1242,112 +1302,113 @@ def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_select_multiples =\ - { - 'children': - { - 'children/fav_colors': - [ - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink' - ], - 'children/ice.creams': - [ - 'children/ice.creams/vanilla', - 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate' - ] - } + expected_select_multiples = { + "children": { + "children/fav_colors": [ + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + ], + "children/ice.creams": [ + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ], } + } select_multiples = export_builder.select_multiples - self.assertTrue('children' in select_multiples) - self.assertTrue('children/fav_colors' in select_multiples['children']) - self.assertTrue('children/ice.creams' in select_multiples['children']) + self.assertTrue("children" in select_multiples) + self.assertTrue("children/fav_colors" in select_multiples["children"]) + self.assertTrue("children/ice.creams" in select_multiples["children"]) self.assertEqual( - sorted([ - choice['xpath'] for choice in - select_multiples['children']['children/fav_colors']]), sorted( - expected_select_multiples['children']['children/fav_colors'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/fav_colors"] + ] + ), + sorted(expected_select_multiples["children"]["children/fav_colors"]), + ) self.assertEqual( - sorted([choice['xpath'] for choice in - select_multiples['children']['children/ice.creams']]), sorted( - expected_select_multiples['children']['children/ice.creams'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/ice.creams"] + ] + ), + sorted(expected_select_multiples["children"]["children/ice.creams"]), + ) def test_split_select_multiples_works(self): """ Test split_select_multiples works as expected. """ - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/fav_colors/red': True, - 'children/fav_colors/blue': True, - 'children/fav_colors/pink': False - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/fav_colors/red": True, + "children/fav_colors/blue": True, + "children/fav_colors/pink": False, + } self.assertEqual(new_row, expected_row) - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + row = { + "children/name": "Mike", + "children/age": 5, + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) def test_split_select_mutliples_works_with_int_value_in_row(self): select_multiples = { - 'children/fav_number': [ + "children/fav_number": [ { - 'xpath': 'children/fav_number/1', - }, { - 'xpath': 'children/fav_number/2', - }, { - 'xpath': 'children/fav_number/3', - } + "xpath": "children/fav_number/1", + }, + { + "xpath": "children/fav_number/2", + }, + { + "xpath": "children/fav_number/3", + }, ] } - row = {'children/fav_number': 1} + row = {"children/fav_number": 1} expected_row = { - 'children/fav_number/1': True, - 'children/fav_number': 1, - 'children/fav_number/3': False, - 'children/fav_number/2': False + "children/fav_number/1": True, + "children/fav_number": 1, + "children/fav_number/3": False, + "children/fav_number/2": False, } new_row = ExportBuilder.split_select_multiples(row, select_multiples) @@ -1355,156 +1416,130 @@ def test_split_select_mutliples_works_with_int_value_in_row(self): self.assertEqual(new_row, expected_row) def test_split_select_multiples_works_when_data_is_blank(self): - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '', - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = {"children/name": "Mike", "children/age": 5, "children/fav_colors": ""} + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "", + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_gps_fields =\ - { - 'childrens_survey': - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', - 'geo/_geolocation_precision' - ] - } + expected_gps_fields = { + "childrens_survey": { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] } + } gps_fields = export_builder.gps_fields - self.assertTrue('childrens_survey' in gps_fields) + self.assertTrue("childrens_survey" in gps_fields) self.assertEqual( - sorted(gps_fields['childrens_survey']), - sorted(expected_gps_fields['childrens_survey'])) + sorted(gps_fields["childrens_survey"]), + sorted(expected_gps_fields["childrens_survey"]), + ) def test_split_gps_components_works(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - 'geo/_geolocation_latitude': '1.0', - 'geo/_geolocation_longitude': '36.1', - 'geo/_geolocation_altitude': '2000', - 'geo/_geolocation_precision': '20' - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "1.0 36.1 2000 20", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "1.0 36.1 2000 20", + "geo/_geolocation_latitude": "1.0", + "geo/_geolocation_longitude": "36.1", + "geo/_geolocation_altitude": "2000", + "geo/_geolocation_precision": "20", + } self.assertEqual(new_row, expected_row) def test_split_gps_components_works_when_gps_data_is_blank(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '', - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "", + } self.assertEqual(new_row, expected_row) def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_encoded_fields =\ - { - 'childrens_survey': - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')), - 'tel/tel.mobile': 'tel/{0}'.format( - _encode_for_mongo('tel.mobile')), - } + expected_encoded_fields = { + "childrens_survey": { + "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")), + "tel/tel.mobile": "tel/{0}".format(_encode_for_mongo("tel.mobile")), } + } encoded_fields = export_builder.encoded_fields - self.assertTrue('childrens_survey' in encoded_fields) + self.assertTrue("childrens_survey" in encoded_fields) self.assertEqual( - encoded_fields['childrens_survey'], - expected_encoded_fields['childrens_survey']) + encoded_fields["childrens_survey"], + expected_encoded_fields["childrens_survey"], + ) def test_decode_fields_names_encoded_for_mongo(self): - encoded_fields = \ - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')) - } - row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/{0}'.format( - _encode_for_mongo('tel.office')): '123-456-789' - } - new_row = ExportBuilder.decode_mongo_encoded_fields( - row, encoded_fields) - expected_row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/tel.office': '123-456-789' - } + encoded_fields = { + "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")) + } + row = { + "name": "Abe", + "age": 35, + "tel/{0}".format(_encode_for_mongo("tel.office")): "123-456-789", + } + new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields) + expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"} self.assertEqual(new_row, expected_row) def test_generate_field_title(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", ".", - data_dictionary=self.dd) + field_name = ExportBuilder.format_field_title( + "children/age", ".", data_dictionary=self.dd + ) expected_field_name = "children.age" self.assertEqual(field_name, expected_field_name) @@ -1513,129 +1548,170 @@ def test_delimiter_replacement_works_existing_fields(self): export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_sections =\ - [ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.name', - 'xpath': 'children/name' - } - ] - } - ] - children_section = export_builder.section_by_name('children') + expected_sections = [ + { + "name": "children", + "elements": [{"title": "children.name", "xpath": "children/name"}], + } + ] + children_section = export_builder.section_by_name("children") self.assertEqual( - children_section['elements'][0]['title'], - expected_sections[0]['elements'][0]['title']) + children_section["elements"][0]["title"], + expected_sections[0]["elements"][0]["title"], + ) def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section =\ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.fav_colors.red', - 'xpath': 'children/fav_colors/red' - } - ] - } - childrens_section = export_builder.section_by_name('children') - match = [x for x in childrens_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "children", + "elements": [ + {"title": "children.fav_colors.red", "xpath": "children/fav_colors/red"} + ], + } + childrens_section = export_builder.section_by_name("children") + match = [ + x + for x in childrens_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section = \ - { - 'name': 'childrens_survey', - 'elements': [ - { - 'title': 'geo._geolocation_latitude', - 'xpath': 'geo/_geolocation_latitude' - } - ] - } - main_section = export_builder.section_by_name('childrens_survey') - match = [x for x in main_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "childrens_survey", + "elements": [ + { + "title": "geo._geolocation_latitude", + "xpath": "geo/_geolocation_latitude", + } + ], + } + main_section = export_builder.section_by_name("childrens_survey") + match = [ + x + for x in main_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xls') + xls_file = NamedTemporaryFile(suffix=".xls") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = xlrd.open_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey', 'children', - 'children_cartoons', - 'children_cartoons_characters'] + expected_sheet_names = [ + "childrens_survey", + "children", + "children_cartoons", + "children_cartoons_characters", + ] self.assertEqual(wb.sheet_names(), expected_sheet_names) # check header columns - main_sheet = wb.sheet_by_name('childrens_survey') + main_sheet = wb.sheet_by_name("childrens_survey") expected_column_headers = [ - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', - 'tel/tel.mobile', '_id', 'meta/instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + "name", + "age", + "geo/geolocation", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "_id", + "meta/instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] column_headers = main_sheet.row_values(0) - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) - childrens_sheet = wb.sheet_by_name('children') + childrens_sheet = wb.sheet_by_name("children") expected_column_headers = [ - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] column_headers = childrens_sheet.row_values(0) - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) - cartoons_sheet = wb.sheet_by_name('children_cartoons') + cartoons_sheet = wb.sheet_by_name("children_cartoons") expected_column_headers = [ - 'children/cartoons/name', 'children/cartoons/why', '_id', - '_uuid', '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + "children/cartoons/name", + "children/cartoons/why", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] column_headers = cartoons_sheet.row_values(0) - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) - characters_sheet = wb.sheet_by_name('children_cartoons_characters') + characters_sheet = wb.sheet_by_name("children_cartoons_characters") expected_column_headers = [ - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] column_headers = characters_sheet.row_values(0) - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close() @@ -1644,94 +1720,115 @@ def test_to_xls_export_respects_custom_field_delimiter(self): export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xls') + xls_file = NamedTemporaryFile(suffix=".xls") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = xlrd.open_workbook(filename) # check header columns - main_sheet = wb.sheet_by_name('childrens_survey') + main_sheet = wb.sheet_by_name("childrens_survey") expected_column_headers = [ - 'name', 'age', 'geo.geolocation', 'geo._geolocation_latitude', - 'geo._geolocation_longitude', 'geo._geolocation_altitude', - 'geo._geolocation_precision', 'tel.tel.office', - 'tel.tel.mobile', '_id', 'meta.instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + "name", + "age", + "geo.geolocation", + "geo._geolocation_latitude", + "geo._geolocation_longitude", + "geo._geolocation_altitude", + "geo._geolocation_precision", + "tel.tel.office", + "tel.tel.mobile", + "_id", + "meta.instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] column_headers = main_sheet.row_values(0) - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close() def test_get_valid_sheet_name_catches_duplicates(self): - work_sheets = {'childrens_survey': "Worksheet"} + work_sheets = {"childrens_survey": "Worksheet"} desired_sheet_name = "childrens_survey" expected_sheet_name = "childrens_survey1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheets) + desired_sheet_name, work_sheets + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_get_valid_sheet_name_catches_long_names(self): desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_lo" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, []) + desired_sheet_name, [] + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_get_valid_sheet_name_catches_long_duplicate_names(self): - work_sheet_titles = ['childrens_survey_with_a_very_lo'] + work_sheet_titles = ["childrens_survey_with_a_very_lo"] desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_l1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheet_titles) + desired_sheet_name, work_sheet_titles + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_to_xls_export_generates_valid_sheet_names(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xls"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xls') + xls_file = NamedTemporaryFile(suffix=".xls") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = xlrd.open_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey_with_a_very_lo', - 'childrens_survey_with_a_very_l1', - 'childrens_survey_with_a_very_l2', - 'childrens_survey_with_a_very_l3'] + expected_sheet_names = [ + "childrens_survey_with_a_very_lo", + "childrens_survey_with_a_very_l1", + "childrens_survey_with_a_very_l2", + "childrens_survey_with_a_very_l3", + ] self.assertEqual(wb.sheet_names(), expected_sheet_names) xls_file.close() def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xls"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') + xls_file = NamedTemporaryFile(suffix=".xlsx") filename = xls_file.name export_builder.to_xls_export(filename, self.long_survey_data) xls_file.seek(0) wb = load_workbook(filename) # get the children's sheet - ws1 = wb['childrens_survey_with_a_very_l1'] + ws1 = wb["childrens_survey_with_a_very_l1"] # parent_table is in cell K2 - parent_table_name = ws1['K2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_lo' + parent_table_name = ws1["K2"].value + expected_parent_table_name = "childrens_survey_with_a_very_lo" self.assertEqual(parent_table_name, expected_parent_table_name) # get cartoons sheet - ws2 = wb['childrens_survey_with_a_very_l2'] - parent_table_name = ws2['G2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_l1' + ws2 = wb["childrens_survey_with_a_very_l2"] + parent_table_name = ws2["G2"].value + expected_parent_table_name = "childrens_survey_with_a_very_l1" self.assertEqual(parent_table_name, expected_parent_table_name) xls_file.close() @@ -1748,15 +1845,12 @@ def test_type_conversion(self): "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", - "_geolocation": [ - "-1.2625482", - "36.7924794" - ], + "_geolocation": ["-1.2625482", "36.7924794"], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", - "really": "15:24:00.000+03" + "really": "15:24:00.000+03", } submission_2 = { @@ -1772,65 +1866,71 @@ def test_type_conversion(self): "amount": "", } - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xls"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} - data = dict_to_joined_export(submission_1, 1, indices, survey_name, - survey, submission_1) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['age'], int) - self.assertIsInstance(new_row['when'], datetime.date) - self.assertIsInstance(new_row['amount'], float) + data = dict_to_joined_export( + submission_1, 1, indices, survey_name, survey, submission_1 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["age"], int) + self.assertIsInstance(new_row["when"], datetime.date) + self.assertIsInstance(new_row["amount"], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} - data = dict_to_joined_export(submission_2, 1, indices, survey_name, - survey, submission_2) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['amount'], basestring) - self.assertEqual(new_row['amount'], '') + data = dict_to_joined_export( + submission_2, 1, indices, survey_name, survey, submission_2 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["amount"], str) + self.assertEqual(new_row["amount"], "") def test_xls_convert_dates_before_1900(self): - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xls"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ { - 'name': 'Abe', - 'when': '1899-07-03', + "name": "Abe", + "when": "1899-07-03", } ] # create export file - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, data) temp_xls_file.close() # this should error if there is a problem, not sure what to assert def test_convert_types(self): - val = '1' + val = "1" expected_val = 1 - converted_val = ExportBuilder.convert_type(val, 'int') + converted_val = ExportBuilder.convert_type(val, "int") self.assertIsInstance(converted_val, int) self.assertEqual(converted_val, expected_val) - val = '1.2' + val = "1.2" expected_val = 1.2 - converted_val = ExportBuilder.convert_type(val, 'decimal') + converted_val = ExportBuilder.convert_type(val, "decimal") self.assertIsInstance(converted_val, float) self.assertEqual(converted_val, expected_val) - val = '2012-06-23' + val = "2012-06-23" expected_val = datetime.date(2012, 6, 23) - converted_val = ExportBuilder.convert_type(val, 'date') + converted_val = ExportBuilder.convert_type(val, "date") self.assertIsInstance(converted_val, datetime.date) self.assertEqual(converted_val, expected_val) @@ -1839,7 +1939,7 @@ def test_to_sav_export(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -1856,47 +1956,47 @@ def test_to_sav_export(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + ) as reader: header = next(reader) rows = [r for r in reader] # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_to_sav_export_language(self): - survey = self._create_childrens_survey('childrens_survey_sw.xls') + survey = self._create_childrens_survey("childrens_survey_sw.xls") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -1913,91 +2013,94 @@ def test_to_sav_export_language(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + ) as reader: header = next(reader) rows = [r for r in reader] - if section != 'childrens_survey_sw': - section += '_sw' + if section != "childrens_survey_sw": + section += "_sw" # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_generate_field_title_truncated_titles(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", "/", - data_dictionary=self.dd, - remove_group_name=True) + field_name = ExportBuilder.format_field_title( + "children/age", "/", data_dictionary=self.dd, remove_group_name=True + ) expected_field_name = "age" self.assertEqual(field_name, expected_field_name) def test_generate_field_title_truncated_titles_select_multiple(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( - "children/fav_colors/red", "/", + "children/fav_colors/red", + "/", data_dictionary=self.dd, - remove_group_name=True + remove_group_name=True, ) expected_field_name = "fav_colors/red" self.assertEqual(field_name, expected_field_name) def test_xls_export_remove_group_name(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertTrue(data[u"fav_colors/red's"]) - self.assertTrue(data[u"fav_colors/blue's"]) - self.assertFalse(data[u"fav_colors/pink's"]) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) temp_xls_file.close() def test_zipped_csv_export_remove_group_name(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2006,100 +2109,109 @@ def test_zipped_csv_export_remove_group_name(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) def test_xls_export_with_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['name.first'], '3.1 Childs name') - self.assertEqual(labels['age'], '3.2 Child age') - self.assertEqual(labels['fav_colors/red\'s'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue\'s'], 'fav_colors/Blue') - self.assertEqual(labels['fav_colors/pink\'s'], 'fav_colors/Pink') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["name.first"], "3.1 Childs name") + self.assertEqual(labels["age"], "3.2 Child age") + self.assertEqual(labels["fav_colors/red's"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue's"], "fav_colors/Blue") + self.assertEqual(labels["fav_colors/pink's"], "fav_colors/Pink") data = dict([(r[0].value, r[2].value) for r in children_sheet.columns]) - self.assertEqual(data['name.first'], 'Mike') - self.assertEqual(data['age'], 5) - self.assertTrue(data['fav_colors/red\'s']) - self.assertTrue(data['fav_colors/blue\'s']) - self.assertFalse(data['fav_colors/pink\'s']) + self.assertEqual(data["name.first"], "Mike") + self.assertEqual(data["age"], 5) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) temp_xls_file.close() def test_xls_export_with_labels_only(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(data['3.1 Childs name'], 'Mike') - self.assertEqual(data['3.2 Child age'], 5) - self.assertTrue(data['fav_colors/Red']) - self.assertTrue(data['fav_colors/Blue']) - self.assertFalse(data['fav_colors/Pink']) + self.assertEqual(data["3.1 Childs name"], "Mike") + self.assertEqual(data["3.2 Child age"], 5) + self.assertTrue(data["fav_colors/Red"]) + self.assertTrue(data["fav_colors/Blue"]) + self.assertFalse(data["fav_colors/Pink"]) temp_xls_file.close() def test_zipped_csv_export_with_labels(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2108,50 +2220,67 @@ def test_zipped_csv_export_with_labels(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - expected_labels = ['3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + expected_labels = [ + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) actual_labels = [h for h in rows[1]] self.assertEqual(sorted(actual_labels), sorted(expected_labels)) data = dict(zip(rows[0], rows[2])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2159,14 +2288,15 @@ def test_zipped_csv_export_with_labels_only(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xls'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xls"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2175,36 +2305,42 @@ def test_zipped_csv_export_with_labels_only(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) expected_headers = [ - '3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by' + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/Red'], 'True') - self.assertEqual(data['fav_colors/Blue'], 'True') - self.assertEqual(data['fav_colors/Pink'], 'False') + self.assertEqual(data["fav_colors/Red"], "True") + self.assertEqual(data["fav_colors/Blue"], "True") + self.assertEqual(data["fav_colors/Pink"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2215,7 +2351,7 @@ def test_to_sav_export_with_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -2232,32 +2368,41 @@ def test_to_sav_export_with_labels(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): sav_path = os.path.join(temp_dir, "{0}.sav".format(section)) - if section == 'children_survey': + if section == "children_survey": with SavHeaderReader(sav_path) as header: expected_labels = [ - '1. What is your name?', '2. How old are you?', - '4. Geo-location', '5.1 Office telephone', - '5.2 Mobile telephone', '_duration', '_id', - '_index', '_notes', '_parent_index', - '_parent_table_name', '_submission_time', - '_submitted_by', - '_tags', '_uuid', '_version', - 'geo/_geolocation_altitude', - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_precision', - 'meta/instanceID' + "1. What is your name?", + "2. How old are you?", + "4. Geo-location", + "5.1 Office telephone", + "5.2 Mobile telephone", + "_duration", + "_id", + "_index", + "_notes", + "_parent_index", + "_parent_table_name", + "_submission_time", + "_submitted_by", + "_tags", + "_uuid", + "_version", + "geo/_geolocation_altitude", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_precision", + "meta/instanceID", ] labels = header.varLabels.values() self.assertEqual(sorted(expected_labels), sorted(labels)) @@ -2267,226 +2412,232 @@ def _test_sav_file(section): rows = [r for r in reader] # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_xls_export_with_english_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_en.xls'), - default_name='childrens_survey_en') - # no default_language is not set - self.assertEqual( - survey.to_json_dict().get('default_language'), 'default' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_en.xls"), + default_name="childrens_survey_en", ) + # no default_language is not set + self.assertEqual(survey.to_json_dict().get("default_language"), "default") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) childrens_survey_sheet = wb["childrens_survey_en"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. What is your name?') - self.assertEqual(labels['age'], '2. How old are you?') + labels = dict( + [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] + ) + self.assertEqual(labels["name"], "1. What is your name?") + self.assertEqual(labels["age"], "2. How old are you?") children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Blue') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["fav_colors/red"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Blue") temp_xls_file.close() def test_xls_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xls'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xls"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) childrens_survey_sheet = wb["childrens_survey_sw"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. Jina lako ni?') - self.assertEqual(labels['age'], '2. Umri wako ni?') + labels = dict( + [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] + ) + self.assertEqual(labels["name"], "1. Jina lako ni?") + self.assertEqual(labels["age"], "2. Umri wako ni?") children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Nyekundu') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Bluu') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["fav_colors/red"], "fav_colors/Nyekundu") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Bluu") temp_xls_file.close() def test_csv_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xls'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xls"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") dd = DataDictionary() dd._survey = survey ordered_columns = OrderedDict() CSVDataFrameBuilder._build_ordered_columns(survey, ordered_columns) - ordered_columns['children/fav_colors/red'] = None - labels = get_labels_from_columns(ordered_columns, dd, '/') - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + ordered_columns["children/fav_colors/red"] = None + labels = get_labels_from_columns(ordered_columns, dd, "/") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) # use language provided in keyword argument - labels = get_labels_from_columns(ordered_columns, dd, '/', - language='english') - self.assertIn('1. What is your name?', labels) - self.assertIn('2. How old are you?', labels) - self.assertIn('fav_colors/Red', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="english") + self.assertIn("1. What is your name?", labels) + self.assertIn("2. How old are you?", labels) + self.assertIn("fav_colors/Red", labels) # use default language when language supplied does not exist - labels = get_labels_from_columns(ordered_columns, dd, '/', - language="Chinese") - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="Chinese") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) def test_select_multiples_choices(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xls'), - default_name='childrens_survey_sw') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xls"), + default_name="childrens_survey_sw", + ) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] self.assertNotEqual(child.children, []) choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'Nyekundu', - '_label_xpath': 'fav_colors/Nyekundu', - 'xpath': 'children/fav_colors/red', - 'title': 'children/fav_colors/red', - 'type': 'string', - 'label': 'fav_colors/Nyekundu' - }, { - '_label': 'Bluu', - '_label_xpath': 'fav_colors/Bluu', - 'xpath': 'children/fav_colors/blue', - 'title': 'children/fav_colors/blue', - 'type': 'string', 'label': 'fav_colors/Bluu' - }, { - '_label': 'Pink', - '_label_xpath': 'fav_colors/Pink', - 'xpath': 'children/fav_colors/pink', - 'title': 'children/fav_colors/pink', - 'type': 'string', 'label': 'fav_colors/Pink' - } + "_label": "Nyekundu", + "_label_xpath": "fav_colors/Nyekundu", + "xpath": "children/fav_colors/red", + "title": "children/fav_colors/red", + "type": "string", + "label": "fav_colors/Nyekundu", + }, + { + "_label": "Bluu", + "_label_xpath": "fav_colors/Bluu", + "xpath": "children/fav_colors/blue", + "title": "children/fav_colors/blue", + "type": "string", + "label": "fav_colors/Bluu", + }, + { + "_label": "Pink", + "_label_xpath": "fav_colors/Pink", + "xpath": "children/fav_colors/pink", + "title": "children/fav_colors/pink", + "type": "string", + "label": "fav_colors/Pink", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'children/fav_colors': [ - ('children/fav_colors/red', 'red', 'Nyekundu'), - ('children/fav_colors/blue', 'blue', 'Bluu'), - ('children/fav_colors/pink', 'pink', 'Pink') - ], 'children/ice.creams': [ - ('children/ice.creams/vanilla', 'vanilla', 'Vanilla'), - ('children/ice.creams/strawberry', 'strawberry', 'Strawberry'), - ('children/ice.creams/chocolate', 'chocolate', 'Chocolate'), - ] + "children/fav_colors": [ + ("children/fav_colors/red", "red", "Nyekundu"), + ("children/fav_colors/blue", "blue", "Bluu"), + ("children/fav_colors/pink", "pink", "Pink"), + ], + "children/ice.creams": [ + ("children/ice.creams/vanilla", "vanilla", "Vanilla"), + ("children/ice.creams/strawberry", "strawberry", "Strawberry"), + ("children/ice.creams/chocolate", "chocolate", "Chocolate"), + ], } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) def test_select_multiples_choices_with_choice_filter(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'choice_filter.xlsx' - ), default_name='choice_filter') + survey = create_survey_from_xls( + _logger_fixture_path("choice_filter.xlsx"), default_name="choice_filter" + ) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'county': [ - ('county/king', 'king', 'King'), - ('county/pierce', 'pierce', 'Pierce'), - ('county/king', 'king', 'King'), - ('county/cameron', 'cameron', 'Cameron') + "county": [ + ("county/king", "king", "King"), + ("county/pierce", "pierce", "Pierce"), + ("county/king", "king", "King"), + ("county/cameron", "cameron", "Cameron"), ] } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) def test_string_to_date_with_xls_validation(self): # test "2016-11-02" @@ -2506,36 +2657,36 @@ def _create_osm_survey(self): """ # publish form - osm_fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'apps', 'api', - 'tests', 'fixtures', 'osm') - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + osm_fixtures_dir = os.path.join( + settings.PROJECT_ROOT, "apps", "api", "tests", "fixtures", "osm" + ) + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) # make submissions filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - paths = [os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") self._make_submission_w_attachment(submission_path, paths) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) return survey def test_zip_csv_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2552,9 +2703,9 @@ def test_zip_csv_export_has_submission_review_fields(self): self.assertIn(REVIEW_DATE, sorted(actual_headers)) self.assertIn(REVIEW_STATUS, sorted(actual_headers)) submission = rows[1] - self.assertEqual(submission[29], 'Rejected') - self.assertEqual(submission[30], 'Wrong Location') - self.assertEqual(submission[31], '2021-05-25T02:27:19') + self.assertEqual(submission[29], "Rejected") + self.assertEqual(submission[30], "Wrong Location") + self.assertEqual(submission[31], "2021-05-25T02:27:19") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2562,13 +2713,13 @@ def test_xls_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in xls exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.osm_data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) @@ -2580,21 +2731,21 @@ def test_xls_export_has_submission_review_fields(self): self.assertIn(REVIEW_COMMENT, sorted(xls_headers)) self.assertIn(REVIEW_DATE, sorted(xls_headers)) self.assertIn(REVIEW_STATUS, sorted(xls_headers)) - self.assertEqual(xls_data[29], 'Rejected') - self.assertEqual(xls_data[30], 'Wrong Location') - self.assertEqual(xls_data[31], '2021-05-25T02:27:19') + self.assertEqual(xls_data[29], "Rejected") + self.assertEqual(xls_data[30], "Wrong Location") + self.assertEqual(xls_data[31], "2021-05-25T02:27:19") def test_zipped_sav_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2603,25 +2754,49 @@ def test_zipped_sav_has_submission_review_fields(self): zip_file.close() temp_zip_file.close() - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_review_comment', f'@{REVIEW_DATE}', '@_review_status', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_review_comment", + f"@{REVIEW_DATE}", + "@_review_status", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], b'Rejected') - self.assertEqual(rows[1][30], b'Wrong Location') - self.assertEqual(rows[1][31], b'2021-05-25T02:27:19') + self.assertEqual(rows[1][29], b"Rejected") + self.assertEqual(rows[1][30], b"Wrong Location") + self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") def test_zipped_csv_export_with_osm_data(self): """ @@ -2631,7 +2806,7 @@ def test_zipped_csv_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2645,24 +2820,44 @@ def test_zipped_csv_export_with_osm_data(self): rows = [row for row in reader] expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], '1424308569120.jpg') - self.assertEqual(rows[1][1], 'OSMWay234134797.osm') - self.assertEqual(rows[1][2], '23.708174238006087') - self.assertEqual(rows[1][4], 'tertiary') - self.assertEqual(rows[1][6], 'Patuatuli Road') - self.assertEqual(rows[1][13], 'kol') + self.assertEqual(rows[1][0], "1424308569120.jpg") + self.assertEqual(rows[1][1], "OSMWay234134797.osm") + self.assertEqual(rows[1][2], "23.708174238006087") + self.assertEqual(rows[1][4], "tertiary") + self.assertEqual(rows[1][6], "Patuatuli Road") + self.assertEqual(rows[1][13], "kol") def test_zipped_sav_export_with_osm_data(self): """ @@ -2670,25 +2865,27 @@ def test_zipped_sav_export_with_osm_data(self): """ survey = self._create_osm_survey() xform = self.xform - osm_data = [{ - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road_ctr_lat': '23.708174238006087', - 'osm_road_ctr_lon': '90.40946505581161', - 'osm_road_highway': 'tertiary', - 'osm_road_lanes': '2', - 'osm_road_name': 'Patuatuli Road', - 'osm_building_building': 'yes', - 'osm_building_building_levels': '4', - 'osm_building_ctr_lat': '23.707316084046038', - 'osm_building_ctr_lon': '90.40849938337506', - 'osm_building_name': 'kol' - }] + osm_data = [ + { + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road_ctr_lat": "23.708174238006087", + "osm_road_ctr_lon": "90.40946505581161", + "osm_road_highway": "tertiary", + "osm_road_lanes": "2", + "osm_road_name": "Patuatuli Road", + "osm_building_building": "yes", + "osm_building_building_levels": "4", + "osm_building_ctr_lat": "23.707316084046038", + "osm_building_ctr_lon": "90.40849938337506", + "osm_building_name": "kol", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2697,27 +2894,49 @@ def test_zipped_sav_export_with_osm_data(self): zip_file.close() temp_zip_file.close() - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], b'1424308569120.jpg') - self.assertEqual(rows[1][1], b'OSMWay234134797.osm') - self.assertEqual(rows[1][2], b'23.708174238006087') - self.assertEqual(rows[1][4], b'tertiary') - self.assertEqual(rows[1][6], b'Patuatuli Road') - self.assertEqual(rows[1][13], b'kol') + self.assertEqual(rows[1][0], b"1424308569120.jpg") + self.assertEqual(rows[1][1], b"OSMWay234134797.osm") + self.assertEqual(rows[1][2], b"23.708174238006087") + self.assertEqual(rows[1][4], b"tertiary") + self.assertEqual(rows[1][6], b"Patuatuli Road") + self.assertEqual(rows[1][13], b"kol") def test_show_choice_labels(self): """ @@ -2735,24 +2954,19 @@ def test_show_choice_labels(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mango']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango"]] self.assertEqual(expected_result, result) @@ -2773,25 +2987,20 @@ def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mangue']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue"]] self.assertEqual(expected_result, result) @@ -2812,26 +3021,20 @@ def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) @@ -2853,26 +3056,20 @@ def test_show_choice_labels_select_multiple_1(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) @@ -2894,29 +3091,24 @@ def test_show_choice_labels_select_multiple_2(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mango', 'fruit/Orange', - 'fruit/Apple'], - ['Maria', 25, 'Mango Orange', 'Mango', 'Orange', None]] + ["name", "age", "fruit", "fruit/Mango", "fruit/Orange", "fruit/Apple"], + ["Maria", 25, "Mango Orange", "Mango", "Orange", None], + ] self.maxDiff = None self.assertEqual(expected_result, result) @@ -2939,27 +3131,21 @@ def test_show_choice_labels_select_multiple_language(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mangue Pomme']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue Pomme"]] self.assertEqual(expected_result, result) @@ -2981,29 +3167,24 @@ def test_show_choice_labels_select_multiple_language_1(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', True, False, True]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", True, False, True], + ] self.assertEqual(expected_result, result) @@ -3026,30 +3207,25 @@ def test_show_choice_labels_select_multiple_language_2(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.BINARY_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 1, 0, 1]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", 1, 0, 1], + ] self.assertEqual(expected_result, result) @@ -3072,30 +3248,25 @@ def test_show_choice_labels_select_multiple_language_3(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 'Mangue', None, 'Pomme']] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", "Mangue", None, "Pomme"], + ] self.assertEqual(expected_result, result) @@ -3128,27 +3299,24 @@ def test_mulsel_export_with_label_choices(self): | | primary | 9 | Other bank | 9 | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - data = [{ - 'banks_deal': '1 2 3 4', - 'primary_bank': '3' - }] # yapf: disable + data = [{"banks_deal": "1 2 3 4", "primary_bank": "3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:2]] - for row in children_sheet.rows] + result = [[col.value for col in row[:2]] for row in children_sheet.rows] expected_result = [ - [u'banks_deal', u'primary_bank'], - [u'KCB Equity Co-operative CBA', u'Co-operative']] + ["banks_deal", "primary_bank"], + ["KCB Equity Co-operative CBA", "Co-operative"], + ] temp_xls_file.close() self.assertEqual(result, expected_result) diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index a411f4ae11..6aacfec896 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -5,7 +5,6 @@ from builtins import str as text from collections import OrderedDict -from past.builtins import basestring from django.db.utils import DataError from django.http import Http404 @@ -13,41 +12,45 @@ from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.xform import XForm -from onadata.libs.data.query import \ - get_form_submissions_aggregated_by_select_one +from onadata.libs.data.query import get_form_submissions_aggregated_by_select_one from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.data.query import get_form_submissions_grouped_by_select_one from onadata.libs.utils import common_tags # list of fields we can chart CHART_FIELDS = [ - 'select one', 'integer', 'decimal', 'date', 'datetime', 'start', 'end', - 'today' + "select one", + "integer", + "decimal", + "date", + "datetime", + "start", + "end", + "today", ] # numeric, categorized DATA_TYPE_MAP = { - 'integer': 'numeric', - 'decimal': 'numeric', - 'datetime': 'time_based', - 'date': 'time_based', - 'start': 'time_based', - 'end': 'time_based', - 'today': 'time_based', - 'calculate': 'numeric', + "integer": "numeric", + "decimal": "numeric", + "datetime": "time_based", + "date": "time_based", + "start": "time_based", + "end": "time_based", + "today": "time_based", + "calculate": "numeric", } FIELD_DATA_MAP = { - common_tags.SUBMISSION_TIME: - ('Submission Time', '_submission_time', 'datetime'), - common_tags.SUBMITTED_BY: ('Submission By', '_submitted_by', 'text'), - common_tags.DURATION: ('Duration', '_duration', 'integer') + common_tags.SUBMISSION_TIME: ("Submission Time", "_submission_time", "datetime"), + common_tags.SUBMITTED_BY: ("Submission By", "_submitted_by", "text"), + common_tags.DURATION: ("Duration", "_duration", "integer"), } CHARTS_PER_PAGE = 20 POSTGRES_ALIAS_LENGTH = 63 -timezone_re = re.compile(r'(.+)\+(\d+)') +timezone_re = re.compile(r"(.+)\+(\d+)") def utc_time_string_for_javascript(date_string): @@ -62,13 +65,13 @@ def utc_time_string_for_javascript(date_string): match = timezone_re.match(date_string) if not match: raise ValueError( - "{} fos not match the format 2014-01-16T12:07:23.322+03".format( - date_string)) + "{} fos not match the format 2014-01-16T12:07:23.322+03".format(date_string) + ) date_time = match.groups()[0] tz = match.groups()[1] if len(tz) == 2: - tz += '00' + tz += "00" elif len(tz) != 4: raise ValueError("len of {} must either be 2 or 4") @@ -77,8 +80,8 @@ def utc_time_string_for_javascript(date_string): def find_choice_label(choices, string): for choice in choices: - if choice['name'] == string: - return choice['label'] + if choice["name"] == string: + return choice["label"] def get_field_choices(field, xform): @@ -88,13 +91,13 @@ def get_field_choices(field, xform): :param xform: :return: Form field choices """ - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") - if isinstance(field, basestring): + if isinstance(field, str): choices = choices.get(field) - elif 'name' in field and field.name in choices: + elif "name" in field and field.name in choices: choices = choices.get(field.name) - elif 'itemset' in field: + elif "itemset" in field: choices = choices.get(field.itemset) return choices @@ -120,9 +123,7 @@ def get_choice_label(choices, string): labels.append(label) else: # Try to get labels by splitting the string - labels = [ - find_choice_label(choices, name) for name in string.split(" ") - ] + labels = [find_choice_label(choices, name) for name in string.split(" ")] # If any split string does not have a label it is not a multiselect # but a missing label, use string @@ -141,34 +142,29 @@ def _flatten_multiple_dict_into_one(field_name, group_by_name, data): # truncate field name to 63 characters to fix #354 truncated_field_name = field_name[0:POSTGRES_ALIAS_LENGTH] truncated_group_by_name = group_by_name[0:POSTGRES_ALIAS_LENGTH] - final = [{ - truncated_field_name: b, - 'items': [] - } for b in list({a.get(truncated_field_name) - for a in data})] + final = [ + {truncated_field_name: b, "items": []} + for b in list({a.get(truncated_field_name) for a in data}) + ] for a in data: for b in final: if a.get(truncated_field_name) == b.get(truncated_field_name): - b['items'].append({ - truncated_group_by_name: - a.get(truncated_group_by_name), - 'count': - a.get('count') - }) + b["items"].append( + { + truncated_group_by_name: a.get(truncated_group_by_name), + "count": a.get("count"), + } + ) return final -def _use_labels_from_field_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_field_name(field_name, field, data_type, data, choices=None): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized' and field_name != common_tags.SUBMITTED_BY: + if data_type == "categorized" and field_name != common_tags.SUBMITTED_BY: if data: if field.children: choices = field.children @@ -176,60 +172,54 @@ def _use_labels_from_field_name(field_name, for item in data: if truncated_name in item: item[truncated_name] = get_choice_label( - choices, item[truncated_name]) + choices, item[truncated_name] + ) for item in data: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def _use_labels_from_group_by_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_group_by_name(field_name, field, data_type, data, choices=None): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized': + if data_type == "categorized": if data: if field.children: choices = field.children for item in data: - if 'items' in item: - for i in item.get('items'): - i[truncated_name] = get_choice_label(choices, - i[truncated_name]) + if "items" in item: + for i in item.get("items"): + i[truncated_name] = get_choice_label(choices, i[truncated_name]) else: - item[truncated_name] = \ - get_choice_label(choices, item[truncated_name]) + item[truncated_name] = get_choice_label( + choices, item[truncated_name] + ) for item in data: - if 'items' in item: - for i in item.get('items'): + if "items" in item: + for i in item.get("items"): if field_name != truncated_name: i[field_name] = i[truncated_name] - del (i[truncated_name]) + del i[truncated_name] else: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def build_chart_data_for_field(xform, - field, - language_index=0, - choices=None, - group_by=None, - data_view=None): +def build_chart_data_for_field( + xform, field, language_index=0, choices=None, group_by=None, data_view=None +): # check if its the special _submission_time META - if isinstance(field, basestring): + if isinstance(field, str): field_label, field_xpath, field_type = FIELD_DATA_MAP.get(field) else: # TODO: merge choices with results and set 0's on any missing fields, @@ -239,96 +229,106 @@ def build_chart_data_for_field(xform, field_xpath = field.get_abbreviated_xpath() field_type = field.type - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') - field_name = field.name if not isinstance(field, basestring) else field + data_type = DATA_TYPE_MAP.get(field_type, "categorized") + field_name = field.name if not isinstance(field, str) else field if group_by and isinstance(group_by, list): group_by_name = [ - g.get_abbreviated_xpath() if not isinstance(g, basestring) else g - for g in group_by + g.get_abbreviated_xpath() if not isinstance(g, str) else g for g in group_by ] result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) elif group_by: - group_by_name = group_by.get_abbreviated_xpath() \ - if not isinstance(group_by, basestring) else group_by - - if (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - isinstance(group_by, six.string_types): + group_by_name = ( + group_by.get_abbreviated_xpath() + if not isinstance(group_by, str) + else group_by + ) + + if ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and isinstance(group_by, six.string_types): result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - elif field_type in common_tags.NUMERIC_LIST and \ - isinstance(group_by, six.string_types): + xform, field_xpath, group_by_name, field_name, data_view + ) + elif field_type in common_tags.NUMERIC_LIST and isinstance( + group_by, six.string_types + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) - elif (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, field_name, group_by_name, data_view + ) + elif ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and group_by.type == common_tags.SELECT_ONE: result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - - result = _flatten_multiple_dict_into_one(field_name, group_by_name, - result) - elif field_type in common_tags.NUMERIC_LIST \ - and group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, group_by_name, field_name, data_view + ) + + result = _flatten_multiple_dict_into_one(field_name, group_by_name, result) + elif ( + field_type in common_tags.NUMERIC_LIST + and group_by.type == common_tags.SELECT_ONE + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) else: - raise ParseError('Cannot group by %s' % group_by_name) + raise ParseError("Cannot group by %s" % group_by_name) else: - result = get_form_submissions_grouped_by_field(xform, field_xpath, - field_name, data_view) + result = get_form_submissions_grouped_by_field( + xform, field_xpath, field_name, data_view + ) result = _use_labels_from_field_name( - field_name, field, data_type, result, choices=choices) + field_name, field, data_type, result, choices=choices + ) - if group_by and not isinstance(group_by, six.string_types + (list, )): - group_by_data_type = DATA_TYPE_MAP.get(group_by.type, 'categorized') + if group_by and not isinstance(group_by, six.string_types + (list,)): + group_by_data_type = DATA_TYPE_MAP.get(group_by.type, "categorized") grp_choices = get_field_choices(group_by, xform) result = _use_labels_from_group_by_name( - group_by_name, - group_by, - group_by_data_type, - result, - choices=grp_choices) + group_by_name, group_by, group_by_data_type, result, choices=grp_choices + ) elif group_by and isinstance(group_by, list): for g in group_by: if isinstance(g, six.string_types): continue - group_by_data_type = DATA_TYPE_MAP.get(g.type, 'categorized') + group_by_data_type = DATA_TYPE_MAP.get(g.type, "categorized") grp_choices = get_field_choices(g, xform) result = _use_labels_from_group_by_name( g.get_abbreviated_xpath(), g, group_by_data_type, result, - choices=grp_choices) + choices=grp_choices, + ) if not group_by: - result = sorted(result, key=lambda d: d['count']) + result = sorted(result, key=lambda d: d["count"]) # for date fields, strip out None values - if data_type == 'time_based': + if data_type == "time_based": result = [r for r in result if r.get(field_name) is not None] # for each check if it matches the timezone regexp and convert for js for r in result: if timezone_re.match(r[field_name]): try: - r[field_name] = utc_time_string_for_javascript( - r[field_name]) + r[field_name] = utc_time_string_for_javascript(r[field_name]) except ValueError: pass return { - 'data': result, - 'data_type': data_type, - 'field_label': field_label, - 'field_xpath': field_xpath, - 'field_name': field_name, - 'field_type': field_type, - 'grouped_by': group_by_name if group_by else None + "data": result, + "data_type": data_type, + "field_label": field_label, + "field_xpath": field_xpath, + "field_name": field_name, + "field_type": field_type, + "grouped_by": group_by_name if group_by else None, } @@ -355,8 +355,7 @@ def build_chart_data(xform, language_index=0, page=0): fields = fields[start:end] return [ - build_chart_data_for_field(xform, field, language_index) - for field in fields + build_chart_data_for_field(xform, field, language_index) for field in fields ] @@ -379,17 +378,15 @@ def build_chart_data_from_widget(widget, language_index=0): fields = [e for e in xform.survey_elements if e.name == field_name] if len(fields) == 0: - raise ParseError("Field %s does not not exist on the form" % - field_name) + raise ParseError("Field %s does not not exist on the form" % field_name) field = fields[0] - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") if choices: choices = choices.get(field_name) try: - data = build_chart_data_for_field( - xform, field, language_index, choices=choices) + data = build_chart_data_for_field(xform, field, language_index, choices=choices) except DataError as e: raise ParseError(text(e)) @@ -408,8 +405,7 @@ def _get_field_from_field_fn(field_str, xform, field_fn): # use specified field to get summary fields = [e for e in xform.survey_elements if field_fn(e) == field_str] if len(fields) == 0: - raise Http404("Field %s does not not exist on the form" % - field_str) + raise Http404("Field %s does not not exist on the form" % field_str) field = fields[0] return field @@ -420,7 +416,8 @@ def get_field_from_field_name(field_name, xform): def get_field_from_field_xpath(field_xpath, xform): return _get_field_from_field_fn( - field_xpath, xform, lambda x: x.get_abbreviated_xpath()) + field_xpath, xform, lambda x: x.get_abbreviated_xpath() + ) def get_field_label(field, language_index=0): @@ -435,12 +432,9 @@ def get_field_label(field, language_index=0): return field_label -def get_chart_data_for_field(field_name, - xform, - accepted_format, - group_by, - field_xpath=None, - data_view=None): +def get_chart_data_for_field( + field_name, xform, accepted_format, group_by, field_xpath=None, data_view=None +): """ Get chart data for a given xlsform field. """ @@ -450,10 +444,9 @@ def get_chart_data_for_field(field_name, field = get_field_from_field_name(field_name, xform) if group_by: - if len(group_by.split(',')) > 1: + if len(group_by.split(",")) > 1: group_by = [ - get_field_from_field_xpath(g, xform) - for g in group_by.split(',') + get_field_from_field_xpath(g, xform) for g in group_by.split(",") ] else: group_by = get_field_from_field_xpath(group_by, xform) @@ -465,21 +458,18 @@ def get_chart_data_for_field(field_name, try: data = build_chart_data_for_field( - xform, - field, - choices=choices, - group_by=group_by, - data_view=data_view) + xform, field, choices=choices, group_by=group_by, data_view=data_view + ) except DataError as e: raise ParseError(text(e)) else: - if accepted_format == 'json' or not accepted_format: + if accepted_format == "json" or not accepted_format: xform = xform.pk - elif accepted_format == 'html' and 'data' in data: - for item in data['data']: + elif accepted_format == "html" and "data" in data: + for item in data["data"]: if isinstance(item[field_name], list): - item[field_name] = ', '.join(item[field_name]) + item[field_name] = ", ".join(item[field_name]) - data.update({'xform': xform}) + data.update({"xform": xform}) return data diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index 4abfece1e0..a6e7a1fd79 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -11,7 +11,6 @@ import traceback import uuid from io import BytesIO -from past.builtins import basestring from django.conf import settings from django.core.mail import mail_admins @@ -21,7 +20,7 @@ import six from raven.contrib.django.raven_compat.models import client -TRUE_VALUES = ['TRUE', 'T', '1', 1] +TRUE_VALUES = ["TRUE", "T", "1", 1] def str_to_bool(str_var): @@ -41,17 +40,16 @@ def get_boolean_value(str_var, default=None): """ Converts a string into boolean """ - if isinstance(str_var, basestring) and \ - str_var.lower() in ['true', 'false']: + if isinstance(str_var, str) and str_var.lower() in ["true", "false"]: return str_to_bool(str_var) return str_var if default else False def get_uuid(hex_only: bool = True): - ''' + """ Return UUID4 hex value - ''' + """ return uuid.uuid4().hex if hex_only else str(uuid.uuid4()) @@ -65,18 +63,19 @@ def report_exception(subject, info, exc_info=None): if exc_info: cls, err = exc_info[:2] - message = _(u"Exception in request:" - u" %(class)s: %(error)s")\ - % {'class': cls.__name__, 'error': err} - message += u"".join(traceback.format_exception(*exc_info)) + message = _("Exception in request:" " %(class)s: %(error)s") % { + "class": cls.__name__, + "error": err, + } + message += "".join(traceback.format_exception(*exc_info)) # send to sentry try: client.captureException(exc_info) except Exception: # pylint: disable=broad-except - logging.exception(_(u'Sending to Sentry failed.')) + logging.exception(_("Sending to Sentry failed.")) else: - message = u"%s" % info + message = "%s" % info if settings.DEBUG or settings.TESTING_MODE: sys.stdout.write("Subject: %s\n" % subject) @@ -89,12 +88,12 @@ def filename_from_disposition(content_disposition): """ Gets a filename from the given content disposition header. """ - filename_pos = content_disposition.index('filename=') + filename_pos = content_disposition.index("filename=") if filename_pos == -1: raise Exception('"filename=" not found in content disposition file') - return content_disposition[filename_pos + len('filename='):] + return content_disposition[filename_pos + len("filename=") :] def get_response_content(response, decode=True): @@ -106,7 +105,7 @@ def get_response_content(response, decode=True): :param response: The response to extract content from. :param decode: If true decode as utf-8, default True. """ - contents = '' + contents = "" if response.streaming: actual_content = BytesIO() for content in response.streaming_content: @@ -117,7 +116,7 @@ def get_response_content(response, decode=True): contents = response.content if decode: - return contents.decode('utf-8') + return contents.decode("utf-8") else: return contents @@ -126,7 +125,7 @@ def json_stream(data, json_string): """ Generator function to stream JSON data """ - yield '[' + yield "[" try: data = data.__iter__() item = next(data) @@ -134,7 +133,7 @@ def json_stream(data, json_string): try: next_item = next(data) yield json_string(item) - yield ',' + yield "," item = next_item except StopIteration: yield json_string(item) @@ -142,7 +141,7 @@ def json_stream(data, json_string): except (AttributeError, StopIteration): pass finally: - yield ']' + yield "]" def retry(tries, delay=3, backoff=2): @@ -181,8 +180,9 @@ def function_retry(self, *args, **kwargs): else: return result # Last ditch effort run against master database - if len(getattr(settings, 'SLAVE_DATABASES', [])): + if len(getattr(settings, "SLAVE_DATABASES", [])): from multidb.pinning import use_master + with use_master: return func(self, *args, **kwargs) @@ -190,11 +190,12 @@ def function_retry(self, *args, **kwargs): return func(self, *args, **kwargs) return function_retry + return decorator_retry def merge_dicts(*dict_args): - """ Given any number of dicts, shallow copy and merge into a new dict, + """Given any number of dicts, shallow copy and merge into a new dict, precedence goes to key value pairs in latter dicts. """ result = {} @@ -206,8 +207,8 @@ def merge_dicts(*dict_args): def cmp_to_key(mycmp): - """ Convert a cmp= function into a key= function - """ + """Convert a cmp= function into a key= function""" + class K(object): def __init__(self, obj, *args): self.obj = obj @@ -229,4 +230,5 @@ def __ge__(self, other): def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 + return K diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index a2023d7597..2a8d99e24c 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -5,43 +5,64 @@ from django.conf import settings from django.db.models.query import QuerySet from django.utils.translation import ugettext as _ -from future.utils import iteritems -from past.builtins import basestring +from six import iteritems from pyxform.question import Question from pyxform.section import RepeatingSection, Section from onadata.apps.logger.models import OsmData from onadata.apps.logger.models.xform import XForm, question_types_to_exclude from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (ParsedInstance, - query_data) +from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, DELETEDAT, DURATION, - EDITED, GEOLOCATION, ID, MEDIA_ALL_RECEIVED, MEDIA_COUNT, NA_REP, - NOTES, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, SELECT_BIND_TYPE, REVIEW_DATE) -from onadata.libs.utils.export_builder import (get_choice_label, - get_value_or_attachment_uri, - track_task_progress) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + GEOLOCATION, + ID, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + NA_REP, + NOTES, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID_STRING, + REVIEW_STATUS, + REVIEW_COMMENT, + MULTIPLE_SELECT_TYPE, + SELECT_BIND_TYPE, + REVIEW_DATE, +) +from onadata.libs.utils.export_builder import ( + get_choice_label, + get_value_or_attachment_uri, + track_task_progress, +) from onadata.libs.utils.model_tools import get_columns_with_hxl # the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = u"select" -GEOPOINT_BIND_TYPE = u"geopoint" +MULTIPLE_SELECT_BIND_TYPE = "select" +GEOPOINT_BIND_TYPE = "geopoint" # column group delimiters -GROUP_DELIMITER_SLASH = '/' -GROUP_DELIMITER_DOT = '.' +GROUP_DELIMITER_SLASH = "/" +GROUP_DELIMITER_DOT = "." DEFAULT_GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] -DEFAULT_NA_REP = getattr(settings, 'NA_REP', NA_REP) +DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) # index tags -DEFAULT_OPEN_TAG = '[' -DEFAULT_CLOSE_TAG = ']' +DEFAULT_OPEN_TAG = "[" +DEFAULT_CLOSE_TAG = "]" DEFAULT_INDEX_TAGS = (DEFAULT_OPEN_TAG, DEFAULT_CLOSE_TAG) YES = 1 @@ -54,25 +75,23 @@ def remove_dups_from_list_maintain_order(lst): def get_prefix_from_xpath(xpath): xpath = str(xpath) - parts = xpath.rsplit('/', 1) + parts = xpath.rsplit("/", 1) if len(parts) == 1: return None elif len(parts) == 2: - return '%s/' % parts[0] + return "%s/" % parts[0] else: - raise ValueError( - '%s cannot be prefixed, it returns %s' % (xpath, str(parts))) + raise ValueError("%s cannot be prefixed, it returns %s" % (xpath, str(parts))) def get_labels_from_columns(columns, dd, group_delimiter, language=None): labels = [] for col in columns: elem = dd.get_survey_element(col) - label = dd.get_label(col, elem=elem, - language=language) if elem else col - if elem is not None and elem.type == '': + label = dd.get_label(col, elem=elem, language=language) if elem else col + if elem is not None and elem.type == "": label = group_delimiter.join([elem.parent.name, label]) - if label == '': + if label == "": label = elem.name labels.append(label) @@ -86,28 +105,35 @@ def get_column_names_only(columns, dd, group_delimiter): elem = dd.get_survey_element(col) if elem is None: new_col = col - elif elem.type != '': + elif elem.type != "": new_col = elem.name else: - new_col = DEFAULT_GROUP_DELIMITER.join([ - elem.parent.name, - elem.name - ]) + new_col = DEFAULT_GROUP_DELIMITER.join([elem.parent.name, elem.name]) new_columns.append(new_col) return new_columns -def write_to_csv(path, rows, columns, columns_with_hxl=None, - remove_group_name=False, dd=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, include_labels=False, - include_labels_only=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, language=None): - na_rep = getattr(settings, 'NA_REP', NA_REP) - encoding = 'utf-8-sig' if win_excel_utf8 else 'utf-8' - with open(path, 'wb') as csvfile: - writer = csv.writer(csvfile, encoding=encoding, lineterminator='\n') +def write_to_csv( + path, + rows, + columns, + columns_with_hxl=None, + remove_group_name=False, + dd=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + include_labels=False, + include_labels_only=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + language=None, +): + na_rep = getattr(settings, "NA_REP", NA_REP) + encoding = "utf-8-sig" if win_excel_utf8 else "utf-8" + with open(path, "wb") as csvfile: + writer = csv.writer(csvfile, encoding=encoding, lineterminator="\n") # Check if to truncate the group name prefix if not include_labels_only: @@ -126,12 +152,13 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, writer.writerow(new_cols) if include_labels or include_labels_only: - labels = get_labels_from_columns(columns, dd, group_delimiter, - language=language) + labels = get_labels_from_columns( + columns, dd, group_delimiter, language=language + ) writer.writerow(labels) if include_hxl and columns_with_hxl: - hxl_row = [columns_with_hxl.get(col, '') for col in columns] + hxl_row = [columns_with_hxl.get(col, "") for col in columns] hxl_row and writer.writerow(hxl_row) for i, row in enumerate(rows, start=1): @@ -142,29 +169,60 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, class AbstractDataFrameBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT, EDITED] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + EDITED, + ] # fields NOT within the form def that we want to include ADDITIONAL_COLUMNS = [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, VERSION, - DURATION, SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED] + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, + ] BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False """ Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML """ - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=True, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=True, include_reviews=False, - language=None): + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=True, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=True, + include_reviews=False, + language=None, + ): self.username = username self.id_string = id_string @@ -176,30 +234,39 @@ def __init__(self, username, id_string, filter_query=None, self.start = start self.end = end self.remove_group_name = remove_group_name - self.extra_columns = ( - self.ADDITIONAL_COLUMNS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.ADDITIONAL_COLUMNS + getattr( + settings, "EXTRA_COLUMNS", [] + ) if include_reviews: self.extra_columns = self.extra_columns + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] if xform: self.xform = xform else: - self.xform = XForm.objects.get(id_string=self.id_string, - user__username=self.username) + self.xform = XForm.objects.get( + id_string=self.id_string, user__username=self.username + ) self.include_labels = include_labels self.include_labels_only = include_labels_only self.include_images = include_images self.include_hxl = include_hxl self.win_excel_utf8 = win_excel_utf8 self.total_records = total_records - if index_tags != DEFAULT_INDEX_TAGS and \ - not isinstance(index_tags, (tuple, list)): - raise ValueError(_( - "Invalid option for repeat_index_tags: %s " - "expecting a tuple with opening and closing tags " - "e.g repeat_index_tags=('[', ']')" % index_tags)) + if index_tags != DEFAULT_INDEX_TAGS and not isinstance( + index_tags, (tuple, list) + ): + raise ValueError( + _( + "Invalid option for repeat_index_tags: %s " + "expecting a tuple with opening and closing tags " + "e.g repeat_index_tags=('[', ']')" % index_tags + ) + ) self.index_tags = index_tags self.show_choice_labels = show_choice_labels self.language = language @@ -208,46 +275,68 @@ def __init__(self, username, id_string, filter_query=None, def _setup(self): self.dd = self.xform - self.select_multiples = self._collect_select_multiples(self.dd, - self.language) + self.select_multiples = self._collect_select_multiples(self.dd, self.language) self.gps_fields = self._collect_gps_fields(self.dd) @classmethod def _fields_to_select(cls, dd): - return [c.get_abbreviated_xpath() - for c in dd.get_survey_elements() if isinstance(c, Question)] + return [ + c.get_abbreviated_xpath() + for c in dd.get_survey_elements() + if isinstance(c, Question) + ] @classmethod def _collect_select_multiples(cls, dd, language=None): select_multiples = [] select_multiple_elements = [ - e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ] for e in select_multiple_elements: xpath = e.get_abbreviated_xpath() - choices = [(c.get_abbreviated_xpath(), c.name, - get_choice_label(c.label, dd, language)) - for c in e.children] + choices = [ + ( + c.get_abbreviated_xpath(), + c.name, + get_choice_label(c.label, dd, language), + ) + for c in e.children + ] is_choice_randomized = str_to_bool( - e.parameters and e.parameters.get('randomize')) - if ((not choices and e.choice_filter) or is_choice_randomized) \ - and e.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(e.itemset) - choices = [(u'/'.join([xpath, i.get('name')]), i.get('name'), - get_choice_label(i.get('label'), dd, language)) - for i in itemset] if itemset else choices + e.parameters and e.parameters.get("randomize") + ) + if ( + (not choices and e.choice_filter) or is_choice_randomized + ) and e.itemset: + itemset = dd.survey.to_json_dict()["choices"].get(e.itemset) + choices = ( + [ + ( + "/".join([xpath, i.get("name")]), + i.get("name"), + get_choice_label(i.get("label"), dd, language), + ) + for i in itemset + ] + if itemset + else choices + ) select_multiples.append((xpath, choices)) return dict(select_multiples) @classmethod - def _split_select_multiples(cls, record, select_multiples, - binary_select_multiples=False, - value_select_multiples=False, - show_choice_labels=False): - """ Prefix contains the xpath and slash if we are within a repeat so + def _split_select_multiples( + cls, + record, + select_multiples, + binary_select_multiples=False, + value_select_multiples=False, + show_choice_labels=False, + ): + """Prefix contains the xpath and slash if we are within a repeat so that we can figure out which select multiples belong to which repeat """ for key, choices in select_multiples.items(): @@ -257,31 +346,59 @@ def _split_select_multiples(cls, record, select_multiples, if key in record: # split selected choices by spaces and join by / to the # element's xpath - selections = ["%s/%s" % (key, r) - for r in record[key].split(" ")] + selections = ["%s/%s" % (key, r) for r in record[key].split(" ")] if value_select_multiples: - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - (label if show_choice_labels else - record[key].split()[selections.index(choice)]) - if choice in selections else None) - for choice, name, label in choices])) + record.update( + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + ( + label + if show_choice_labels + else record[key].split()[ + selections.index(choice) + ] + ) + if choice in selections + else None, + ) + for choice, name, label in choices + ] + ) + ) elif not binary_select_multiples: # add columns to record for every choice, with default # False and set to True for items in selections - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - choice in selections) - for choice, name, label in choices])) + record.update( + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + choice in selections, + ) + for choice, name, label in choices + ] + ) + ) else: record.update( - dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - YES if choice in selections else NO) - for choice, name, label in choices])) + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + YES if choice in selections else NO, + ) + for choice, name, label in choices + ] + ) + ) # remove the column since we are adding separate columns # for each choice record.pop(key) @@ -292,40 +409,45 @@ def _split_select_multiples(cls, record, select_multiples, for list_item in record_item: if isinstance(list_item, dict): cls._split_select_multiples( - list_item, select_multiples, + list_item, + select_multiples, binary_select_multiples=binary_select_multiples, # noqa value_select_multiples=value_select_multiples, - show_choice_labels=show_choice_labels) + show_choice_labels=show_choice_labels, + ) return record @classmethod def _collect_gps_fields(cls, dd): - return [e.get_abbreviated_xpath() for e in dd.get_survey_elements() - if e.bind.get("type") == "geopoint"] + return [ + e.get_abbreviated_xpath() + for e in dd.get_survey_elements() + if e.bind.get("type") == "geopoint" + ] @classmethod def _tag_edit_string(cls, record): """ Turns a list of tags into a string representation. """ - if '_tags' in record: + if "_tags" in record: tags = [] - for tag in record['_tags']: - if ',' in tag and ' ' in tag: + for tag in record["_tags"]: + if "," in tag and " " in tag: tags.append('"%s"' % tag) else: tags.append(tag) - record.update({'_tags': u', '.join(sorted(tags))}) + record.update({"_tags": ", ".join(sorted(tags))}) @classmethod def _split_gps_fields(cls, record, gps_fields): updated_gps_fields = {} for (key, value) in iteritems(record): - if key in gps_fields and isinstance(value, basestring): + if key in gps_fields and isinstance(value, str): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key) gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) # hack, check if its a list and grab the object within that - parts = value.split(' ') + parts = value.split(" ") # TODO: check whether or not we can have a gps recording # from ODKCollect that has less than four components, # for now we are assuming that this is not the case. @@ -339,19 +461,24 @@ def _split_gps_fields(cls, record, gps_fields): cls._split_gps_fields(list_item, gps_fields) record.update(updated_gps_fields) - def _query_data(self, query='{}', start=0, - limit=ParsedInstance.DEFAULT_LIMIT, - fields='[]', count=False): + def _query_data( + self, + query="{}", + start=0, + limit=ParsedInstance.DEFAULT_LIMIT, + fields="[]", + count=False, + ): # query_data takes params as json strings # so we dumps the fields dictionary count_args = { - 'xform': self.xform, - 'query': query, - 'start': self.start, - 'end': self.end, - 'fields': '[]', - 'sort': '{}', - 'count': True + "xform": self.xform, + "query": query, + "start": self.start, + "end": self.end, + "fields": "[]", + "sort": "{}", + "count": True, } count_object = list(query_data(**count_args)) record_count = count_object[0]["count"] @@ -362,17 +489,17 @@ def _query_data(self, query='{}', start=0, return record_count else: query_args = { - 'xform': self.xform, - 'query': query, - 'fields': fields, - 'start': self.start, - 'end': self.end, + "xform": self.xform, + "query": query, + "fields": fields, + "start": self.start, + "end": self.end, # TODO: we might want to add this in for the user # to sepcify a sort order - 'sort': 'id', - 'start_index': start, - 'limit': limit, - 'count': False + "sort": "id", + "start_index": start, + "limit": limit, + "count": False, } cursor = query_data(**query_args) @@ -380,49 +507,88 @@ def _query_data(self, query='{}', start=0, class CSVDataFrameBuilder(AbstractDataFrameBuilder): - - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=False, include_reviews=False, - language=None): + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=False, + include_reviews=False, + language=None, + ): super(CSVDataFrameBuilder, self).__init__( - username, id_string, filter_query, group_delimiter, - split_select_multiples, binary_select_multiples, start, end, - remove_group_name, xform, include_labels, include_labels_only, - include_images, include_hxl, win_excel_utf8, total_records, - index_tags, value_select_multiples, - show_choice_labels, include_reviews, language) + username, + id_string, + filter_query, + group_delimiter, + split_select_multiples, + binary_select_multiples, + start, + end, + remove_group_name, + xform, + include_labels, + include_labels_only, + include_images, + include_hxl, + win_excel_utf8, + total_records, + index_tags, + value_select_multiples, + show_choice_labels, + include_reviews, + language, + ) self.ordered_columns = OrderedDict() - self.image_xpaths = [] if not self.include_images \ - else self.dd.get_media_survey_xpaths() + self.image_xpaths = ( + [] if not self.include_images else self.dd.get_media_survey_xpaths() + ) def _setup(self): super(CSVDataFrameBuilder, self)._setup() @classmethod - def _reindex(cls, key, value, ordered_columns, row, data_dictionary, - parent_prefix=None, - include_images=True, split_select_multiples=True, - index_tags=DEFAULT_INDEX_TAGS, show_choice_labels=False, - language=None): + def _reindex( + cls, + key, + value, + ordered_columns, + row, + data_dictionary, + parent_prefix=None, + include_images=True, + split_select_multiples=True, + index_tags=DEFAULT_INDEX_TAGS, + show_choice_labels=False, + language=None, + ): """ Flatten list columns by appending an index, otherwise return as is """ + def get_ordered_repeat_value(xpath, repeat_value): """ Return OrderedDict of repeats in the order in which they appear in the XForm. """ - children = data_dictionary.get_child_elements( - xpath, split_select_multiples) + children = data_dictionary.get_child_elements(xpath, split_select_multiples) item = OrderedDict() for elem in children: @@ -435,8 +601,11 @@ def get_ordered_repeat_value(xpath, repeat_value): d = {} # check for lists - if isinstance(value, list) and len(value) > 0 \ - and key not in [ATTACHMENTS, NOTES]: + if ( + isinstance(value, list) + and len(value) > 0 + and key not in [ATTACHMENTS, NOTES] + ): for index, item in enumerate(value): # start at 1 index += 1 @@ -452,69 +621,95 @@ def get_ordered_repeat_value(xpath, repeat_value): # "children/details/immunization/polio_1", # generate ["children", index, "immunization/polio_1"] if parent_prefix is not None: - _key = '/'.join( - parent_prefix + - key.split('/')[len(parent_prefix):]) - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=_key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(_key.split('/')):] + _key = "/".join( + parent_prefix + key.split("/")[len(parent_prefix) :] + ) + xpaths = [ + "{key}{open_tag}{index}{close_tag}".format( + key=_key, + open_tag=index_tags[0], + index=index, + close_tag=index_tags[1], + ) + ] + nested_key.split("/")[len(_key.split("/")) :] else: - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(key.split('/')):] + xpaths = [ + "{key}{open_tag}{index}{close_tag}".format( + key=key, + open_tag=index_tags[0], + index=index, + close_tag=index_tags[1], + ) + ] + nested_key.split("/")[len(key.split("/")) :] # re-create xpath the split on / xpaths = "/".join(xpaths).split("/") new_prefix = xpaths[:-1] if isinstance(nested_val, list): # if nested_value is a list, rinse and repeat - d.update(cls._reindex( - nested_key, nested_val, - ordered_columns, row, data_dictionary, - new_prefix, - include_images=include_images, - split_select_multiples=split_select_multiples, - index_tags=index_tags, - show_choice_labels=show_choice_labels, - language=language)) + d.update( + cls._reindex( + nested_key, + nested_val, + ordered_columns, + row, + data_dictionary, + new_prefix, + include_images=include_images, + split_select_multiples=split_select_multiples, + index_tags=index_tags, + show_choice_labels=show_choice_labels, + language=language, + ) + ) else: # it can only be a scalar # collapse xpath - new_xpath = u"/".join(xpaths) + new_xpath = "/".join(xpaths) # check if this key exists in our ordered columns if key in list(ordered_columns): if new_xpath not in ordered_columns[key]: ordered_columns[key].append(new_xpath) d[new_xpath] = get_value_or_attachment_uri( - nested_key, nested_val, row, data_dictionary, + nested_key, + nested_val, + row, + data_dictionary, include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, + key, + value, + row, + data_dictionary, + include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: # anything that's not a list will be in the top level dict so its # safe to simply assign if key == NOTES: # Do not include notes - d[key] = u"" + d[key] = "" else: d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, - show_choice_labels=show_choice_labels, language=language) + key, + value, + row, + data_dictionary, + include_images, + show_choice_labels=show_choice_labels, + language=language, + ) return d @classmethod - def _build_ordered_columns(cls, survey_element, ordered_columns, - is_repeating_section=False): + def _build_ordered_columns( + cls, survey_element, ordered_columns, is_repeating_section=False + ): """ Build a flat ordered dict of column groups @@ -528,11 +723,12 @@ def _build_ordered_columns(cls, survey_element, ordered_columns, if isinstance(child, RepeatingSection): ordered_columns[child.get_abbreviated_xpath()] = [] child_is_repeating = True - cls._build_ordered_columns(child, ordered_columns, - child_is_repeating) - elif isinstance(child, Question) and not \ - question_types_to_exclude(child.type) and not\ - is_repeating_section: # if is_repeating_section, + cls._build_ordered_columns(child, ordered_columns, child_is_repeating) + elif ( + isinstance(child, Question) + and not question_types_to_exclude(child.type) + and not is_repeating_section + ): # if is_repeating_section, # its parent already initiliased an empty list # so we dont add it to our list of columns, # the repeating columns list will be @@ -550,11 +746,14 @@ def _update_ordered_columns_from_data(self, cursor): for (key, choices) in iteritems(self.select_multiples): # HACK to ensure choices are NOT duplicated if key in self.ordered_columns.keys(): - self.ordered_columns[key] = \ - remove_dups_from_list_maintain_order( - [choice.replace('/' + name, '/' + label) - if self.show_choice_labels else choice - for choice, name, label in choices]) + self.ordered_columns[key] = remove_dups_from_list_maintain_order( + [ + choice.replace("/" + name, "/" + label) + if self.show_choice_labels + else choice + for choice, name, label in choices + ] + ) # add ordered columns for gps fields for key in self.gps_fields: @@ -566,12 +765,17 @@ def _update_ordered_columns_from_data(self, cursor): # re index column repeats for (key, value) in iteritems(record): self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.dd, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) def _format_for_dataframe(self, cursor): """ @@ -581,10 +785,12 @@ def _format_for_dataframe(self, cursor): # split select multiples if self.split_select_multiples: record = self._split_select_multiples( - record, self.select_multiples, + record, + self.select_multiples, self.BINARY_SELECT_MULTIPLES, self.VALUE_SELECT_MULTIPLES, - show_choice_labels=self.show_choice_labels) + show_choice_labels=self.show_choice_labels, + ) # check for gps and split into # components i.e. latitude, longitude, # altitude, precision @@ -594,12 +800,17 @@ def _format_for_dataframe(self, cursor): # re index repeats for (key, value) in iteritems(record): reindexed = self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.dd, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) flat_dict.update(reindexed) yield flat_dict @@ -608,8 +819,9 @@ def export_to(self, path, dataview=None): self._build_ordered_columns(self.dd.survey, self.ordered_columns) if dataview: - cursor = dataview.query_data(dataview, all_data=True, - filter_query=self.filter_query) + cursor = dataview.query_data( + dataview, all_data=True, filter_query=self.filter_query + ) if isinstance(cursor, QuerySet): cursor = cursor.iterator() @@ -617,11 +829,15 @@ def export_to(self, path, dataview=None): data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns) - if [c for c in dataview.columns if xpath.startswith(c)]] - )) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + if [c for c in dataview.columns if xpath.startswith(c)] + ] + ) + ) else: try: cursor = self._query_data(self.filter_query) @@ -637,29 +853,40 @@ def export_to(self, path, dataview=None): # Unpack xform columns and data data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns)])) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + ] + ) + ) # add extra columns columns += [col for col in self.extra_columns] - for field in self.dd.get_survey_elements_of_type('osm'): - columns += OsmData.get_tag_keys(self.xform, - field.get_abbreviated_xpath(), - include_prefix=True) + for field in self.dd.get_survey_elements_of_type("osm"): + columns += OsmData.get_tag_keys( + self.xform, field.get_abbreviated_xpath(), include_prefix=True + ) columns_with_hxl = self.include_hxl and get_columns_with_hxl( - self.dd.survey_elements) - - write_to_csv(path, data, columns, - columns_with_hxl=columns_with_hxl, - remove_group_name=self.remove_group_name, - dd=self.dd, group_delimiter=self.group_delimiter, - include_labels=self.include_labels, - include_labels_only=self.include_labels_only, - include_hxl=self.include_hxl, - win_excel_utf8=self.win_excel_utf8, - total_records=self.total_records, - index_tags=self.index_tags, - language=self.language) + self.dd.survey_elements + ) + + write_to_csv( + path, + data, + columns, + columns_with_hxl=columns_with_hxl, + remove_group_name=self.remove_group_name, + dd=self.dd, + group_delimiter=self.group_delimiter, + include_labels=self.include_labels, + include_labels_only=self.include_labels_only, + include_hxl=self.include_hxl, + win_excel_utf8=self.win_excel_utf8, + total_records=self.total_records, + index_tags=self.index_tags, + language=self.language, + ) diff --git a/onadata/libs/utils/dict_tools.py b/onadata/libs/utils/dict_tools.py index f57c87c450..466ae6820f 100644 --- a/onadata/libs/utils/dict_tools.py +++ b/onadata/libs/utils/dict_tools.py @@ -4,8 +4,6 @@ """ import json -from past.builtins import basestring - def get_values_matching_key(doc, key): """ @@ -37,7 +35,7 @@ def list_to_dict(items, value): key = items.pop() result = {} - bracket_index = key.find('[') + bracket_index = key.find("[") if bracket_index > 0: value = [value] @@ -58,15 +56,17 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): for row in list_of_dicts: for k, v in row.items(): if isinstance(v, list): - z = merge_list_of_dicts(result[k] + v if k in result else v, - override_keys=override_keys) + z = merge_list_of_dicts( + result[k] + v if k in result else v, override_keys=override_keys + ) result[k] = z if isinstance(z, list) else [z] else: if k in result: if isinstance(v, dict): try: result[k] = merge_list_of_dicts( - [result[k], v], override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) except AttributeError as e: # If the key is within the override_keys # (Is a select_multiple question) We make @@ -74,12 +74,15 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): # more accurate as they usually mean that # the select_multiple has been split into # separate columns for each choice - if override_keys and isinstance(result[k], str)\ - and k in override_keys: + if ( + override_keys + and isinstance(result[k], str) + and k in override_keys + ): result[k] = {} result[k] = merge_list_of_dicts( - [result[k], v], - override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) else: raise e else: @@ -95,11 +98,11 @@ def remove_indices_from_dict(obj): Removes indices from a obj dict. """ if not isinstance(obj, dict): - raise ValueError(u"Expecting a dict, found: {}".format(type(obj))) + raise ValueError("Expecting a dict, found: {}".format(type(obj))) result = {} for key, val in obj.items(): - bracket_index = key.find('[') + bracket_index = key.find("[") key = key[:bracket_index] if bracket_index > -1 else key val = remove_indices_from_dict(val) if isinstance(val, dict) else val if isinstance(val, list): @@ -126,7 +129,7 @@ def csv_dict_to_nested_dict(csv_dict, select_multiples=None): for key in list(csv_dict): result = {} value = csv_dict[key] - split_keys = key.split('/') + split_keys = key.split("/") if len(split_keys) == 1: result[key] = value @@ -147,8 +150,8 @@ def dict_lists2strings(adict): :param d: The dict to convert. :returns: The converted dict.""" for k, v in adict.items(): - if isinstance(v, list) and all([isinstance(e, basestring) for e in v]): - adict[k] = ' '.join(v) + if isinstance(v, list) and all([isinstance(e, str) for e in v]): + adict[k] = " ".join(v) elif isinstance(v, dict): adict[k] = dict_lists2strings(v) @@ -162,8 +165,8 @@ def dict_paths2dict(adict): result = {} for k, v in adict.items(): - if k.find('/') > 0: - parts = k.split('/') + if k.find("/") > 0: + parts = k.split("/") if len(parts) > 1: k = parts[0] for part in parts[1:]: @@ -181,7 +184,7 @@ def query_list_to_dict(query_list_str): data_list = json.loads(query_list_str) data_dict = dict() for value in data_list: - data_dict[value['label']] = value['text'] + data_dict[value["label"]] = value["text"] return data_dict @@ -190,7 +193,7 @@ def floip_response_headers_dict(data, xform_headers): """ Returns a dict from matching xform headers and floip responses. """ - headers = [i.split('/')[-1] for i in xform_headers] + headers = [i.split("/")[-1] for i in xform_headers] data = [i[4] for i in data] flow_dict = dict(zip(headers, data)) diff --git a/onadata/libs/utils/qrcode.py b/onadata/libs/utils/qrcode.py index 7fade1e99c..f82673b917 100644 --- a/onadata/libs/utils/qrcode.py +++ b/onadata/libs/utils/qrcode.py @@ -1,28 +1,41 @@ +# -*- coding: utf-8 -*- +""" +QR code utility function. +""" from base64 import b64encode from elaphe import barcode from io import BytesIO -from past.builtins import basestring -def generate_qrcode(message, stream=None, - eclevel='M', margin=10, - data_mode='8bits', format='PNG', scale=2.5): - """ Generate a QRCode, settings options and output.""" +def generate_qrcode( + message, + stream=None, + eclevel="M", + margin=10, + data_mode="8bits", + format="PNG", + scale=2.5, +): + """Generate a QRCode, settings options and output.""" if stream is None: stream = BytesIO() - if isinstance(message, basestring): + if isinstance(message, str): message = message.encode() - img = barcode('qrcode', message, - options=dict(version=9, eclevel=eclevel), - margin=margin, data_mode=data_mode, scale=scale) + img = barcode( + "qrcode", + message, + options=dict(version=9, eclevel=eclevel), + margin=margin, + data_mode=data_mode, + scale=scale, + ) img.save(stream, format) - datauri = "data:image/png;base64,%s"\ - % b64encode(stream.getvalue()).decode("utf-8") + datauri = "data:image/png;base64,%s" % b64encode(stream.getvalue()).decode("utf-8") stream.close() return datauri diff --git a/onadata/libs/utils/string.py b/onadata/libs/utils/string.py index 13d8f44105..1ece191bc0 100644 --- a/onadata/libs/utils/string.py +++ b/onadata/libs/utils/string.py @@ -1,6 +1,13 @@ -from past.builtins import basestring +# -*- coding: utf-8 -*- +""" +String utility function str2bool - converts yes, true, t, 1 to True +else returns the argument value v. +""" def str2bool(v): - return v.lower() in ( - 'yes', 'true', 't', '1') if isinstance(v, basestring) else v + """ + String utility function str2bool - converts "yes", "true", "t", "1" to True + else returns the argument value v. + """ + return v.lower() in ("yes", "true", "t", "1") if isinstance(v, str) else v diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 62db834666..deaec9556f 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -6,13 +6,12 @@ import sys import zipfile from builtins import open -from future.utils import iteritems from json.decoder import JSONDecodeError from tempfile import NamedTemporaryFile from typing import Dict from xml.dom import minidom -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings from django.core.files.storage import get_storage_class @@ -24,16 +23,14 @@ from onadata.libs.utils.common_tags import EXPORT_MIMES from onadata.libs.utils.common_tools import report_exception -SLASH = u"/" +SLASH = "/" def image_urls_for_form(xform): """Return image urls of all image attachments of the xform.""" return sum( - [ - image_urls(s) - for s in xform.instances.filter(deleted_at__isnull=True) - ], []) + [image_urls(s) for s in xform.instances.filter(deleted_at__isnull=True)], [] + ) def get_path(path, suffix): @@ -52,7 +49,7 @@ def image_urls(instance): """ default_storage = get_storage_class()() urls = [] - suffix = settings.THUMB_CONF['medium']['suffix'] + suffix = settings.THUMB_CONF["medium"]["suffix"] for attachment in instance.attachments.all(): path = get_path(attachment.media_file.name, suffix) if default_storage.exists(path): @@ -76,14 +73,15 @@ def parse_xform_instance(xml_str): # THIS IS OKAY FOR OUR USE CASE, BUT OTHER USERS SHOULD BEWARE. survey_data = dict(_path_value_pairs(root_node)) if len(list(_all_attributes(root_node))) != 1: - raise AssertionError(_( - u"There should be exactly one attribute in this document.")) - survey_data.update({ - common_tags.XFORM_ID_STRING: - root_node.getAttribute(u"id"), - common_tags.INSTANCE_DOC_NAME: - root_node.nodeName, - }) + raise AssertionError( + _("There should be exactly one attribute in this document.") + ) + survey_data.update( + { + common_tags.XFORM_ID_STRING: root_node.getAttribute("id"), + common_tags.INSTANCE_DOC_NAME: root_node.nodeName, + } + ) return survey_data @@ -105,8 +103,7 @@ def _path_value_pairs(node): if node.childNodes: # there's no data for this leaf node yield _path(node), None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node yield _path(node), node.childNodes[0].nodeValue else: @@ -130,7 +127,7 @@ def django_file(path, field_name, content_type): """Return an InMemoryUploadedFile object for file uploads.""" # adapted from here: http://groups.google.com/group/django-users/browse_th\ # read/thread/834f988876ff3c45/ - file_object = open(path, 'rb') + file_object = open(path, "rb") return InMemoryUploadedFile( file=file_object, @@ -138,7 +135,8 @@ def django_file(path, field_name, content_type): name=file_object.name, content_type=content_type, size=os.path.getsize(path), - charset=None) + charset=None, + ) def export_def_from_filename(filename): @@ -156,38 +154,38 @@ def get_client_ip(request): arguments: request -- HttpRequest object. """ - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: - return x_forwarded_for.split(',')[0] + return x_forwarded_for.split(",")[0] - return request.META.get('REMOTE_ADDR') + return request.META.get("REMOTE_ADDR") -def get_enketo_urls(form_url, - id_string, - instance_xml=None, - instance_id=None, - return_url=None, - **kwargs) -> Dict[str, str]: +def get_enketo_urls( + form_url, id_string, instance_xml=None, instance_id=None, return_url=None, **kwargs +) -> Dict[str, str]: """Return Enketo URLs.""" - if (not hasattr(settings, 'ENKETO_URL') or - not hasattr(settings, 'ENKETO_API_ALL_SURVEY_LINKS_PATH') or - not hasattr(settings, 'ENKETO_API_TOKEN') or - settings.ENKETO_API_TOKEN == ''): + if ( + not hasattr(settings, "ENKETO_URL") + or not hasattr(settings, "ENKETO_API_ALL_SURVEY_LINKS_PATH") + or not hasattr(settings, "ENKETO_API_TOKEN") + or settings.ENKETO_API_TOKEN == "" + ): return False - url = urljoin( - settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) + url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) - values = {'form_id': id_string, 'server_url': form_url} + values = {"form_id": id_string, "server_url": form_url} if instance_id is not None and instance_xml is not None: url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_INSTANCE_PATH) - values.update({ - 'instance': instance_xml, - 'instance_id': instance_id, - # convert to unicode string in python3 compatible way - 'return_url': u'%s' % return_url - }) + values.update( + { + "instance": instance_xml, + "instance_id": instance_id, + # convert to unicode string in python3 compatible way + "return_url": "%s" % return_url, + } + ) if kwargs: # Kwargs need to take note of xform variable paths i.e. @@ -197,11 +195,15 @@ def get_enketo_urls(form_url, response = requests.post( url, data=values, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) resp_content = response.content - resp_content = resp_content.decode('utf-8') if hasattr( - resp_content, 'decode') else resp_content + resp_content = ( + resp_content.decode("utf-8") + if hasattr(resp_content, "decode") + else resp_content + ) if response.status_code in [200, 201]: try: data = json.loads(resp_content) @@ -219,16 +221,17 @@ def handle_enketo_error(response): try: data = json.loads(response.content) except (ValueError, JSONDecodeError): - report_exception("HTTP Error {}".format(response.status_code), - response.text, sys.exc_info()) + report_exception( + "HTTP Error {}".format(response.status_code), response.text, sys.exc_info() + ) if response.status_code == 502: raise EnketoError( - u"Sorry, we cannot load your form right now. Please try " - "again later.") + "Sorry, we cannot load your form right now. Please try " "again later." + ) raise EnketoError() else: - if 'message' in data: - raise EnketoError(data['message']) + if "message" in data: + raise EnketoError(data["message"]) raise EnketoError(response.text) @@ -237,7 +240,7 @@ def generate_enketo_form_defaults(xform, **kwargs): defaults = {} if kwargs: - for (name, value) in iteritems(kwargs): + for (name, value) in kwargs.items(): field = xform.get_survey_element(name) if field: defaults["defaults[{}]".format(field.get_xpath())] = value @@ -249,7 +252,7 @@ def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file tmp = NamedTemporaryFile() - with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name @@ -261,7 +264,8 @@ def create_attachments_zipfile(attachments): report_exception( "Create attachment zip exception", "File is greater than {} bytes".format( - settings.ZIP_REPORT_ATTACHMENT_LIMIT) + settings.ZIP_REPORT_ATTACHMENT_LIMIT + ), ) break else: @@ -281,8 +285,8 @@ def get_form(kwargs): from onadata.apps.logger.models import XForm from django.http import Http404 - queryset = kwargs.pop('queryset', XForm.objects.all()) - kwargs['deleted_at__isnull'] = True + queryset = kwargs.pop("queryset", XForm.objects.all()) + kwargs["deleted_at__isnull"] = True xform = queryset.filter(**kwargs).first() if xform: return xform @@ -290,12 +294,14 @@ def get_form(kwargs): raise Http404("XForm does not exist.") -def get_form_url(request, - username=None, - protocol='https', - preview=False, - xform_pk=None, - generate_consistent_urls=False): +def get_form_url( + request, + username=None, + protocol="https", + preview=False, + xform_pk=None, + generate_consistent_urls=False, +): """ Return a form list url endpoint to be used to make a request to Enketo. @@ -309,17 +315,18 @@ def get_form_url(request, http_host = settings.TEST_HTTP_HOST username = settings.TEST_USERNAME else: - http_host = request.META.get('HTTP_HOST', 'ona.io') + http_host = request.META.get("HTTP_HOST", "ona.io") - url = '%s://%s' % (protocol, http_host) + url = "%s://%s" % (protocol, http_host) if preview: - url += '/preview' + url += "/preview" if xform_pk and generate_consistent_urls: url += "/enketo/{}".format(xform_pk) elif username: - url += "/{}/{}".format(username, xform_pk) if xform_pk \ - else "/{}".format(username) + url += ( + "/{}/{}".format(username, xform_pk) if xform_pk else "/{}".format(username) + ) return url diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 22a6756ebe..5637eeff09 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -19,19 +19,17 @@ from celery.signals import after_setup_logger from django.core.exceptions import SuspiciousOperation from django.utils.log import AdminEmailHandler -from past.builtins import basestring # setting default encoding to utf-8 -if sys.version[0] == '2': +if sys.version[0] == "2": reload(sys) sys.setdefaultencoding("utf-8") CURRENT_FILE = os.path.abspath(__file__) -PROJECT_ROOT = os.path.realpath( - os.path.join(os.path.dirname(CURRENT_FILE), '../')) +PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(CURRENT_FILE), "../")) PRINT_EXCEPTION = False -TEMPLATED_EMAIL_TEMPLATE_DIR = 'templated_email/' +TEMPLATED_EMAIL_TEMPLATE_DIR = "templated_email/" ADMINS = ( # ('Your Name', 'your_email@example.com'), @@ -39,9 +37,9 @@ MANAGERS = ADMINS -DEFAULT_FROM_EMAIL = 'noreply@ona.io' -SHARE_PROJECT_SUBJECT = '{} Ona Project has been shared with you.' -SHARE_ORG_SUBJECT = '{}, You have been added to {} organisation.' +DEFAULT_FROM_EMAIL = "noreply@ona.io" +SHARE_PROJECT_SUBJECT = "{} Ona Project has been shared with you." +SHARE_ORG_SUBJECT = "{}, You have been added to {} organisation." DEFAULT_SESSION_EXPIRY_TIME = 21600 # 6 hours DEFAULT_TEMP_TOKEN_EXPIRY_TIME = 21600 # 6 hours @@ -52,21 +50,21 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'America/New_York' +TIME_ZONE = "America/New_York" # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" LANGUAGES = ( - ('fr', u'Français'), - ('en', u'English'), - ('es', u'Español'), - ('it', u'Italiano'), - ('km', u'ភាសាខ្មែរ'), - ('ne', u'नेपाली'), - ('nl', u'Nederlands'), - ('zh', u'中文'), + ("fr", "Français"), + ("en", "English"), + ("es", "Español"), + ("it", "Italiano"), + ("km", "ភាសាខ្មែរ"), + ("ne", "नेपाली"), + ("nl", "Nederlands"), + ("zh", "中文"), ) SITE_ID = 1 @@ -82,39 +80,39 @@ # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = 'http://localhost:8000/media/' +MEDIA_URL = "http://localhost:8000/media/" # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') +STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") # URL prefix for static files. # Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Enketo URL -ENKETO_PROTOCOL = 'https' -ENKETO_URL = 'https://enketo.ona.io/' -ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' -ENKETO_API_INSTANCE_PATH = '/api_v2/instance' -ENKETO_API_TOKEN = '' +ENKETO_PROTOCOL = "https" +ENKETO_URL = "https://enketo.ona.io/" +ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" +ENKETO_API_INSTANCE_PATH = "/api_v2/instance" +ENKETO_API_TOKEN = "" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v2/instance/iframe" -ENKETO_API_SALT = 'secretsalt' +ENKETO_API_SALT = "secretsalt" VERIFY_SSL = True -ENKETO_AUTH_COOKIE = '__enketo' -ENKETO_META_UID_COOKIE = '__enketo_meta_uid' -ENKETO_META_USERNAME_COOKIE = '__enketo_meta_username' +ENKETO_AUTH_COOKIE = "__enketo" +ENKETO_META_UID_COOKIE = "__enketo_meta_uid" +ENKETO_META_USERNAME_COOKIE = "__enketo_meta_username" # Login URLs -LOGIN_URL = '/accounts/login/' -LOGIN_REDIRECT_URL = '/login_redirect/' +LOGIN_URL = "/accounts/login/" +LOGIN_REDIRECT_URL = "/login_redirect/" # URL prefix for admin static files -- CSS, JavaScript and images. # Make sure to use a trailing slash. # Examples: "http://foo.com/static/admin/", "/static/admin/". -ADMIN_MEDIA_PREFIX = '/static/admin/' +ADMIN_MEDIA_PREFIX = "/static/admin/" # Additional locations of static files STATICFILES_DIRS = ( @@ -126,29 +124,29 @@ # List of finder classes that know how to find static files in # various locations. STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(PROJECT_ROOT, 'libs/templates'), + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(PROJECT_ROOT, "libs/templates"), ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.template.context_processors.tz', - 'django.contrib.messages.context_processors.messages', - 'onadata.apps.main.context_processors.google_analytics', - 'onadata.apps.main.context_processors.site_name', + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages", + "onadata.apps.main.context_processors.google_analytics", + "onadata.apps.main.context_processors.site_name", ], }, }, @@ -156,80 +154,81 @@ MIDDLEWARE = ( - 'onadata.libs.profiling.sql.SqlTimingMiddleware', - 'django.middleware.http.ConditionalGetMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', + "onadata.libs.profiling.sql.SqlTimingMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks', - 'django.middleware.csrf.CsrfViewMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', - 'onadata.libs.utils.middleware.OperationalErrorMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks", + "django.middleware.csrf.CsrfViewMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", + "onadata.libs.utils.middleware.OperationalErrorMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ) -X_FRAME_OPTIONS = 'DENY' +X_FRAME_OPTIONS = "DENY" -LOCALE_PATHS = (os.path.join(PROJECT_ROOT, 'onadata.apps.main', 'locale'), ) +LOCALE_PATHS = (os.path.join(PROJECT_ROOT, "onadata.apps.main", "locale"),) -ROOT_URLCONF = 'onadata.apps.main.urls' +ROOT_URLCONF = "onadata.apps.main.urls" USE_TZ = True # needed by guardian -ANONYMOUS_DEFAULT_USERNAME = 'AnonymousUser' +ANONYMOUS_DEFAULT_USERNAME = "AnonymousUser" INSTALLED_APPS = ( - 'django.contrib.contenttypes', - 'django.contrib.auth', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.humanize', - 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.gis', - 'registration', - 'django_nose', - 'django_digest', - 'corsheaders', - 'oauth2_provider', - 'rest_framework', - 'rest_framework.authtoken', - 'taggit', - 'onadata.apps.logger', - 'onadata.apps.viewer', - 'onadata.apps.main', - 'onadata.apps.restservice', - 'onadata.apps.api', - 'guardian', - 'onadata.apps.sms_support', - 'onadata.libs', - 'reversion', - 'actstream', - 'onadata.apps.messaging.apps.MessagingConfig', - 'django_filters', - 'oidc', + "django.contrib.contenttypes", + "django.contrib.auth", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.humanize", + "django.contrib.admin", + "django.contrib.admindocs", + "django.contrib.gis", + "registration", + "django_nose", + "django_digest", + "corsheaders", + "oauth2_provider", + "rest_framework", + "rest_framework.authtoken", + "taggit", + "onadata.apps.logger", + "onadata.apps.viewer", + "onadata.apps.main", + "onadata.apps.restservice", + "onadata.apps.api", + "guardian", + "onadata.apps.sms_support", + "onadata.libs", + "reversion", + "actstream", + "onadata.apps.messaging.apps.MessagingConfig", + "django_filters", + "oidc", ) OAUTH2_PROVIDER = { # this is the list of available scopes - 'SCOPES': { - 'read': 'Read scope', - 'write': 'Write scope', - 'groups': 'Access to your groups'}, - 'OAUTH2_VALIDATOR_CLASS': 'onadata.libs.authentication.MasterReplicaOAuth2Validator' # noqa + "SCOPES": { + "read": "Read scope", + "write": "Write scope", + "groups": "Access to your groups", + }, + "OAUTH2_VALIDATOR_CLASS": "onadata.libs.authentication.MasterReplicaOAuth2Validator", # noqa } OPENID_CONNECT_VIEWSET_CONFIG = { "REDIRECT_AFTER_AUTH": "http://localhost:3000", "USE_SSO_COOKIE": True, "SSO_COOKIE_DATA": "email", - "JWT_SECRET_KEY": 'thesecretkey', - "JWT_ALGORITHM": 'HS256', + "JWT_SECRET_KEY": "thesecretkey", + "JWT_ALGORITHM": "HS256", "SSO_COOKIE_MAX_AGE": None, "SSO_COOKIE_DOMAIN": "localhost", "USE_AUTH_BACKEND": False, @@ -248,68 +247,62 @@ "REDIRECT_URI": "http://localhost:8000/oidc/msft/callback", "RESPONSE_TYPE": "id_token", "RESPONSE_MODE": "form_post", - "USE_NONCES": True + "USE_NONCES": True, } } REST_FRAMEWORK = { # Use hyperlinked styles by default. # Only used if the `serializer_class` attribute is not set on a view. - 'DEFAULT_MODEL_SERIALIZER_CLASS': - 'rest_framework.serializers.HyperlinkedModelSerializer', - + "DEFAULT_MODEL_SERIALIZER_CLASS": "rest_framework.serializers.HyperlinkedModelSerializer", # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.AllowAny', + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.AllowAny", ], - 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'onadata.libs.authentication.DigestAuthentication', - 'onadata.libs.authentication.TempTokenAuthentication', - 'onadata.libs.authentication.EnketoTokenAuthentication', - 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', - 'rest_framework.authentication.SessionAuthentication', - 'rest_framework.authentication.TokenAuthentication', + "DEFAULT_AUTHENTICATION_CLASSES": ( + "onadata.libs.authentication.DigestAuthentication", + "onadata.libs.authentication.TempTokenAuthentication", + "onadata.libs.authentication.EnketoTokenAuthentication", + "oauth2_provider.contrib.rest_framework.OAuth2Authentication", + "rest_framework.authentication.SessionAuthentication", + "rest_framework.authentication.TokenAuthentication", ), - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework_jsonp.renderers.JSONPRenderer', - 'rest_framework_csv.renderers.CSVRenderer', + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "rest_framework_jsonp.renderers.JSONPRenderer", + "rest_framework_csv.renderers.CSVRenderer", ), } SWAGGER_SETTINGS = { - "exclude_namespaces": [], # List URL namespaces to ignore - "api_version": '1.0', # Specify your API's version (optional) - "enabled_methods": [ # Methods to enable in UI - 'get', - 'post', - 'put', - 'patch', - 'delete' + "exclude_namespaces": [], # List URL namespaces to ignore + "api_version": "1.0", # Specify your API's version (optional) + "enabled_methods": [ # Methods to enable in UI + "get", + "post", + "put", + "patch", + "delete", ], } CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_CREDENTIALS = True -CORS_ORIGIN_WHITELIST = ( - 'http://dev.ona.io', -) -CORS_URLS_ALLOW_ALL_REGEX = ( - r'^/api/v1/osm/.*$', -) +CORS_ORIGIN_WHITELIST = ("http://dev.ona.io",) +CORS_URLS_ALLOW_ALL_REGEX = (r"^/api/v1/osm/.*$",) USE_THOUSAND_SEPARATOR = True COMPRESS = True # extra data stored with users -AUTH_PROFILE_MODULE = 'onadata.apps.main.UserProfile' +AUTH_PROFILE_MODULE = "onadata.apps.main.UserProfile" # case insensitive usernames AUTHENTICATION_BACKENDS = ( - 'onadata.apps.main.backends.ModelBackend', - 'guardian.backends.ObjectPermissionBackend', + "onadata.apps.main.backends.ModelBackend", + "guardian.backends.ObjectPermissionBackend", ) # Settings for Django Registration @@ -340,55 +333,49 @@ def skip_suspicious_operations(record): # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s' + - ' %(process)d %(thread)d %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "%(levelname)s %(asctime)s %(module)s" + + " %(process)d %(thread)d %(message)s" }, - 'profiler': { - 'format': '%(levelname)s %(asctime)s %(message)s' + "simple": {"format": "%(levelname)s %(message)s"}, + "profiler": {"format": "%(levelname)s %(asctime)s %(message)s"}, + "sql": { + "format": "%(levelname)s %(process)d %(thread)d" + + " %(time)s seconds %(message)s %(sql)s" }, - 'sql': { - 'format': '%(levelname)s %(process)d %(thread)d' + - ' %(time)s seconds %(message)s %(sql)s' + "sql_totals": { + "format": "%(levelname)s %(process)d %(thread)d %(time)s seconds" + + " %(message)s %(num_queries)s sql queries" }, - 'sql_totals': { - 'format': '%(levelname)s %(process)d %(thread)d %(time)s seconds' + - ' %(message)s %(num_queries)s sql queries' - } }, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - }, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, # Define filter for suspicious urls - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "skip_suspicious_operations": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_suspicious_operations, }, }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false', 'skip_suspicious_operations'], - 'class': 'django.utils.log.AdminEmailHandler' + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false", "skip_suspicious_operations"], + "class": "django.utils.log.AdminEmailHandler", }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'verbose', - 'stream': sys.stdout + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + "stream": sys.stdout, }, - 'audit': { - 'level': 'DEBUG', - 'class': 'onadata.libs.utils.log.AuditLogHandler', - 'formatter': 'verbose', - 'model': 'onadata.apps.main.models.audit.AuditLog' + "audit": { + "level": "DEBUG", + "class": "onadata.libs.utils.log.AuditLogHandler", + "formatter": "verbose", + "model": "onadata.apps.main.models.audit.AuditLog", }, # 'sql_handler': { # 'level': 'DEBUG', @@ -403,22 +390,18 @@ def skip_suspicious_operations(record): # 'stream': sys.stdout # } }, - 'loggers': { - 'django.request': { - 'handlers': ['mail_admins', 'console'], - 'level': 'DEBUG', - 'propagate': True, + "loggers": { + "django.request": { + "handlers": ["mail_admins", "console"], + "level": "DEBUG", + "propagate": True, }, - 'console_logger': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': True - }, - 'audit_logger': { - 'handlers': ['audit'], - 'level': 'DEBUG', - 'propagate': True + "console_logger": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": True, }, + "audit_logger": {"handlers": ["audit"], "level": "DEBUG", "propagate": True}, # 'sql_logger': { # 'handlers': ['sql_handler'], # 'level': 'DEBUG', @@ -429,12 +412,12 @@ def skip_suspicious_operations(record): # 'level': 'DEBUG', # 'propagate': True # } - } + }, } # PROFILE_API_ACTION_FUNCTION is used to toggle profiling a viewset's action PROFILE_API_ACTION_FUNCTION = False -PROFILE_LOG_BASE = '/tmp/' +PROFILE_LOG_BASE = "/tmp/" def configure_logging(logger, **kwargs): @@ -445,24 +428,24 @@ def configure_logging(logger, **kwargs): after_setup_logger.connect(configure_logging) -GOOGLE_STEP2_URI = 'http://ona.io/gwelcome' -GOOGLE_OAUTH2_CLIENT_ID = 'REPLACE ME' -GOOGLE_OAUTH2_CLIENT_SECRET = 'REPLACE ME' +GOOGLE_STEP2_URI = "http://ona.io/gwelcome" +GOOGLE_OAUTH2_CLIENT_ID = "REPLACE ME" +GOOGLE_OAUTH2_CLIENT_SECRET = "REPLACE ME" THUMB_CONF = { - 'large': {'size': 1280, 'suffix': '-large'}, - 'medium': {'size': 640, 'suffix': '-medium'}, - 'small': {'size': 240, 'suffix': '-small'}, + "large": {"size": 1280, "suffix": "-large"}, + "medium": {"size": 640, "suffix": "-medium"}, + "small": {"size": 240, "suffix": "-small"}, } # order of thumbnails from largest to smallest -THUMB_ORDER = ['large', 'medium', 'small'] -DEFAULT_IMG_FILE_TYPE = 'jpg' +THUMB_ORDER = ["large", "medium", "small"] +DEFAULT_IMG_FILE_TYPE = "jpg" # celery CELERY_TASK_ALWAYS_EAGER = False CELERY_TASK_IGNORE_RESULT = False CELERY_TASK_TRACK_STARTED = True -CELERY_IMPORTS = ('onadata.libs.utils.csv_import',) +CELERY_IMPORTS = ("onadata.libs.utils.csv_import",) CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD = 100000 # Bytes @@ -479,12 +462,12 @@ def configure_logging(logger, **kwargs): # default content length for submission requests DEFAULT_CONTENT_LENGTH = 10000000 -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' -NOSE_ARGS = ['--with-fixture-bundling', '--nologcapture', '--nocapture'] +TEST_RUNNER = "django_nose.NoseTestSuiteRunner" +NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] # fake endpoints for testing -TEST_HTTP_HOST = 'testserver.com' -TEST_USERNAME = 'bob' +TEST_HTTP_HOST = "testserver.com" +TEST_USERNAME = "bob" # specify the root folder which may contain a templates folder and a static # folder used to override templates for site specific details @@ -494,46 +477,46 @@ def configure_logging(logger, **kwargs): BINARY_SELECT_MULTIPLES = False # Use 'n/a' for empty values by default on csv exports -NA_REP = 'n/a' +NA_REP = "n/a" -if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, basestring): +if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, str): # site templates overrides - TEMPLATES[0]['DIRS'] = [ - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'templates'), - ] + TEMPLATES[0]['DIRS'] + TEMPLATES[0]["DIRS"] = [ + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "templates"), + ] + TEMPLATES[0]["DIRS"] # site static files path STATICFILES_DIRS += ( - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'static'), + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "static"), ) # Set wsgi url scheme to HTTPS -os.environ['wsgi.url_scheme'] = 'https' +os.environ["wsgi.url_scheme"] = "https" SUPPORTED_MEDIA_UPLOAD_TYPES = [ - 'audio/mp3', - 'audio/mpeg', - 'audio/wav', - 'audio/x-m4a', - 'image/jpeg', - 'image/png', - 'image/svg+xml', - 'text/csv', - 'text/json', - 'video/3gpp', - 'video/mp4', - 'application/json', - 'application/pdf', - 'application/msword', - 'application/vnd.ms-excel', - 'application/vnd.ms-powerpoint', - 'application/vnd.oasis.opendocument.text', - 'application/vnd.oasis.opendocument.spreadsheet', - 'application/vnd.oasis.opendocument.presentation', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.openxmlformats-officedocument.presentationml.\ - presentation', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/zip', + "audio/mp3", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", + "image/jpeg", + "image/png", + "image/svg+xml", + "text/csv", + "text/json", + "video/3gpp", + "video/mp4", + "application/json", + "application/pdf", + "application/msword", + "application/vnd.ms-excel", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.presentationml.\ + presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/zip", ] CSV_ROW_IMPORT_ASYNC_THRESHOLD = 100 @@ -543,27 +526,29 @@ def configure_logging(logger, **kwargs): PARSED_INSTANCE_DEFAULT_LIMIT = 1000000 PARSED_INSTANCE_DEFAULT_BATCHSIZE = 1000 -PROFILE_SERIALIZER = \ +PROFILE_SERIALIZER = ( "onadata.libs.serializers.user_profile_serializer.UserProfileSerializer" -ORG_PROFILE_SERIALIZER = \ +) +ORG_PROFILE_SERIALIZER = ( "onadata.libs.serializers.organization_serializer.OrganizationSerializer" +) BASE_VIEWSET = "onadata.libs.baseviewset.DefaultBaseViewset" path = os.path.join(PROJECT_ROOT, "..", "extras", "reserved_accounts.txt") EXPORT_WITH_IMAGE_DEFAULT = True try: - with open(path, 'r') as f: + with open(path, "r") as f: RESERVED_USERNAMES = [line.rstrip() for line in f] except EnvironmentError: RESERVED_USERNAMES = [] -STATIC_DOC = '/static/docs/index.html' +STATIC_DOC = "/static/docs/index.html" try: HOSTNAME = socket.gethostname() except Exception: - HOSTNAME = 'localhost' + HOSTNAME = "localhost" CACHE_MIXIN_SECONDS = 60 @@ -583,17 +568,17 @@ def configure_logging(logger, **kwargs): # email verification ENABLE_EMAIL_VERIFICATION = False -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -XLS_EXTENSIONS = ['xls', 'xlsx'] +XLS_EXTENSIONS = ["xls", "xlsx"] -CSV_EXTENSION = 'csv' +CSV_EXTENSION = "csv" PROJECT_QUERY_CHUNK_SIZE = 5000 # Prevents "The number of GET/POST parameters exceeded" exception DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000 -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # Time in minutes to lock out user from account LOCKOUT_TIME = 30 * 60 From d5506bc4c37fb5c00c0de7258579b52288082fe5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 04:55:44 +0300 Subject: [PATCH 003/234] Switch to using six instead of future --- .../apps/api/models/organization_profile.py | 85 +- onadata/apps/api/models/team.py | 33 +- onadata/apps/api/models/temp_token.py | 10 +- .../tests/viewsets/test_project_viewset.py | 2231 ++++++++--------- .../viewsets/test_user_profile_viewset.py | 1294 +++++----- onadata/apps/api/tools.py | 297 ++- .../apps/api/viewsets/user_profile_viewset.py | 2 +- onadata/apps/api/viewsets/xform_viewset.py | 737 +++--- onadata/apps/logger/models/data_view.py | 296 ++- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/logger/models/open_data.py | 15 +- onadata/apps/logger/models/project.py | 149 +- onadata/apps/logger/models/survey_type.py | 5 +- onadata/apps/logger/models/xform.py | 647 ++--- .../logger/tests/test_briefcase_client.py | 89 +- onadata/apps/logger/xform_instance_parser.py | 82 +- onadata/apps/main/forms.py | 348 +-- onadata/apps/main/models/user_profile.py | 80 +- onadata/apps/main/tests/test_base.py | 272 +- .../apps/main/tests/test_form_enter_data.py | 2 +- onadata/apps/main/tests/test_process.py | 553 ++-- onadata/apps/main/tests/test_user_settings.py | 26 +- onadata/apps/restservice/models.py | 52 +- onadata/apps/restservice/services/textit.py | 20 +- onadata/apps/viewer/models/data_dictionary.py | 148 +- onadata/apps/viewer/models/export.py | 110 +- onadata/apps/viewer/parsed_instance_tools.py | 83 +- onadata/apps/viewer/tasks.py | 154 +- onadata/apps/viewer/tests/test_kml_export.py | 41 +- onadata/libs/renderers/renderers.py | 226 +- .../libs/serializers/attachment_serializer.py | 55 +- .../libs/serializers/metadata_serializer.py | 280 ++- .../serializers/password_reset_serializer.py | 93 +- .../libs/serializers/project_serializer.py | 360 +-- onadata/libs/serializers/widget_serializer.py | 89 +- onadata/libs/serializers/xform_serializer.py | 411 +-- onadata/libs/tests/utils/test_email.py | 125 +- onadata/libs/utils/briefcase_client.py | 144 +- onadata/libs/utils/csv_import.py | 416 +-- onadata/libs/utils/decorators.py | 14 +- onadata/libs/utils/email.py | 68 +- onadata/libs/utils/export_builder.py | 1072 ++++---- onadata/libs/utils/export_tools.py | 511 ++-- onadata/libs/utils/gravatar.py | 14 +- onadata/libs/utils/osm.py | 109 +- onadata/settings/common.py | 2 + requirements/base.in | 14 +- requirements/base.pip | 121 +- setup.cfg | 2 +- 49 files changed, 6522 insertions(+), 5467 deletions(-) diff --git a/onadata/apps/api/models/organization_profile.py b/onadata/apps/api/models/organization_profile.py index fc58c95b81..50179b9208 100644 --- a/onadata/apps/api/models/organization_profile.py +++ b/onadata/apps/api/models/organization_profile.py @@ -6,7 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models.signals import post_delete, post_save -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -23,7 +23,7 @@ def org_profile_post_delete_callback(sender, instance, **kwargs): """ # delete the org_user too instance.user.delete() - safe_delete('{}{}'.format(IS_ORG, instance.pk)) + safe_delete("{}{}".format(IS_ORG, instance.pk)) def create_owner_team_and_assign_permissions(org): @@ -32,14 +32,13 @@ def create_owner_team_and_assign_permissions(org): assigns the group and user permissions """ team = Team.objects.create( - name=Team.OWNER_TEAM_NAME, organization=org.user, - created_by=org.created_by) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + name=Team.OWNER_TEAM_NAME, organization=org.user, created_by=org.created_by + ) + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") # pylint: disable=unpacking-non-sequence permission, _ = Permission.objects.get_or_create( - codename="is_org_owner", name="Organization Owner", - content_type=content_type) # pylint: disable= + codename="is_org_owner", name="Organization Owner", content_type=content_type + ) # pylint: disable= team.permissions.add(permission) org.creator.groups.add(team) @@ -53,23 +52,14 @@ def create_owner_team_and_assign_permissions(org): assign_perm(perm.codename, org.created_by, org) if org.userprofile_ptr: - for perm in get_perms_for_model( - org.userprofile_ptr.__class__): - assign_perm( - perm.codename, org.user, org.userprofile_ptr) + for perm in get_perms_for_model(org.userprofile_ptr.__class__): + assign_perm(perm.codename, org.user, org.userprofile_ptr) if org.creator: - assign_perm( - perm.codename, - org.creator, - org.userprofile_ptr) - - if org.created_by and\ - org.created_by != org.creator: - assign_perm( - perm.codename, - org.created_by, - org.userprofile_ptr) + assign_perm(perm.codename, org.creator, org.userprofile_ptr) + + if org.created_by and org.created_by != org.creator: + assign_perm(perm.codename, org.created_by, org.userprofile_ptr) return team @@ -88,20 +78,20 @@ class OrganizationProfile(UserProfile): """Organization: Extends the user profile for organization specific info - * What does this do? - - it has a createor - - it has owner(s), through permissions/group - - has members, through permissions/group - - no login access, no password? no registration like a normal user? - - created by a user who becomes the organization owner - * What relationships? + * What does this do? + - it has a createor + - it has owner(s), through permissions/group + - has members, through permissions/group + - no login access, no password? no registration like a normal user? + - created by a user who becomes the organization owner + * What relationships? """ class Meta: - app_label = 'api' + app_label = "api" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to an organization") + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to an organization"), ) is_organization = models.BooleanField(default=True) @@ -109,7 +99,7 @@ class Meta: creator = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return "%s[%s]" % (self.name, self.user.username) def save(self, *args, **kwargs): # pylint: disable=arguments-differ super(OrganizationProfile, self).save(*args, **kwargs) @@ -119,7 +109,7 @@ def remove_user_from_organization(self, user): :param user: The user to remove from this organization. """ - for group in user.groups.filter('%s#' % self.user.username): + for group in user.groups.filter("%s#" % self.user.username): user.groups.remove(group) def is_organization_owner(self, user): @@ -130,27 +120,32 @@ def is_organization_owner(self, user): :returns: Boolean whether user has organization level permissions. """ has_owner_group = user.groups.filter( - name='%s#%s' % (self.user.username, Team.OWNER_TEAM_NAME)) + name="%s#%s" % (self.user.username, Team.OWNER_TEAM_NAME) + ) return True if has_owner_group else False post_save.connect( - _post_save_create_owner_team, sender=OrganizationProfile, - dispatch_uid='create_owner_team_and_permissions') + _post_save_create_owner_team, + sender=OrganizationProfile, + dispatch_uid="create_owner_team_and_permissions", +) -post_delete.connect(org_profile_post_delete_callback, - sender=OrganizationProfile, - dispatch_uid='org_profile_post_delete_callback') +post_delete.connect( + org_profile_post_delete_callback, + sender=OrganizationProfile, + dispatch_uid="org_profile_post_delete_callback", +) # pylint: disable=model-no-explicit-unicode class OrgProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) class OrgProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index 0297fceb8b..4cb3f99c51 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -1,4 +1,4 @@ -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from django.db import models from django.db.models.signals import post_save @@ -16,24 +16,28 @@ class Team(Group): we should remove them from all teams and projects within the organization. """ + class Meta: - app_label = 'api' + app_label = "api" OWNER_TEAM_NAME = "Owners" organization = models.ForeignKey(User, on_delete=models.CASCADE) projects = models.ManyToManyField(Project) created_by = models.ForeignKey( - User, related_name='team_creator', null=True, blank=True, - on_delete=models.SET_NULL) + User, + related_name="team_creator", + null=True, + blank=True, + on_delete=models.SET_NULL, + ) - date_created = models.DateTimeField(auto_now_add=True, null=True, - blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) def __str__(self): # return a clear group name without username to user for viewing - return self.name.split('#')[1] + return self.name.split("#")[1] @property def team_name(self): @@ -42,8 +46,8 @@ def team_name(self): def save(self, *args, **kwargs): # allow use of same name in different organizations/users # concat with # - if not self.name.startswith('#'.join([self.organization.username])): - self.name = u'%s#%s' % (self.organization.username, self.name) + if not self.name.startswith("#".join([self.organization.username])): + self.name = "%s#%s" % (self.organization.username, self.name) super(Team, self).save(*args, **kwargs) @@ -55,12 +59,15 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if instance.created_by: assign_perm(perm.codename, instance.created_by, instance) - if hasattr(organization, 'creator') and \ - organization.creator != instance.created_by: + if ( + hasattr(organization, "creator") + and organization.creator != instance.created_by + ): assign_perm(perm.codename, organization.creator, instance) if organization.created_by != instance.created_by: assign_perm(perm.codename, organization.created_by, instance) -post_save.connect(set_object_permissions, sender=Team, - dispatch_uid='set_team_object_permissions') +post_save.connect( + set_object_permissions, sender=Team, dispatch_uid="set_team_object_permissions" +) diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index 0de4b6cc42..3f46fc53ae 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -7,9 +7,9 @@ from django.conf import settings from django.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible -AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') +AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") @python_2_unicode_compatible @@ -18,13 +18,15 @@ class TempToken(models.Model): """ The temporary authorization token model. """ + key = models.CharField(max_length=40, primary_key=True) user = models.OneToOneField( - AUTH_USER_MODEL, related_name='_user', on_delete=models.CASCADE) + AUTH_USER_MODEL, related_name="_user", on_delete=models.CASCADE + ) created = models.DateTimeField(auto_now_add=True) class Meta: - app_label = 'api' + app_label = "api" def save(self, *args, **kwargs): if not self.key: diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index f55ab99ed3..16c0da382d 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -5,7 +5,7 @@ import json import os from builtins import str -from future.utils import iteritems +from six import iteritems from operator import itemgetter from collections import OrderedDict @@ -20,11 +20,11 @@ import requests from onadata.apps.api import tools -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.team_viewset import TeamViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet @@ -33,29 +33,40 @@ from onadata.libs import permissions as role from onadata.libs.models.share_project import ShareProject from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key -from onadata.libs.permissions import (ROLES_ORDERED, DataEntryMinorRole, - DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, - OwnerRole, ReadOnlyRole, - ReadOnlyRoleNoDownload) -from onadata.libs.serializers.project_serializer import (BaseProjectSerializer, - ProjectSerializer) - -ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryRole, - EditorRole, - ManagerRole, - OwnerRole] - - -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +from onadata.libs.permissions import ( + ROLES_ORDERED, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRole, + ReadOnlyRoleNoDownload, +) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) + +ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, +] + + +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock(url, request): response = requests.Response() response.status_code = 201 - response._content = \ - '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n'\ - ' "code": "200"\n}' + response._content = ( + '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n' ' "code": "200"\n}' + ) return response @@ -65,38 +76,38 @@ def get_latest_tags(project): class TestProjectViewSet(TestAbstractViewSet): - def setUp(self): super(TestProjectViewSet, self).setUp() - self.view = ProjectViewSet.as_view({ - 'get': 'list', - 'post': 'create' - }) + self.view = ProjectViewSet.as_view({"get": "list", "post": "create"}) def tearDown(self): cache.clear() super(TestProjectViewSet, self).tearDown() - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.urlopen") def test_publish_xlsform_using_url_upload(self, mock_urlopen): with HTTMock(enketo_mock): self._project_create() - view = ProjectViewSet.as_view({ - 'post': 'forms' - }) + view = ProjectViewSet.as_view({"post": "forms"}) pre_count = XForm.objects.count() project_id = self.project.pk - xls_url = 'https://ona.io/examples/forms/tutorial/form.xlsx' + xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation_different_id_string.xlsx") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation_different_id_string.xlsx", + ) + + xls_file = open(path, "rb") mock_urlopen.return_value = xls_file - post_data = {'xls_url': xls_url} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) mock_urlopen.assert_called_with(xls_url) @@ -105,61 +116,63 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): self.assertEqual(XForm.objects.count(), pre_count + 1) self.assertEqual( XFormVersion.objects.filter( - xform__pk=response.data.get('formid')).count(), 1) + xform__pk=response.data.get("formid") + ).count(), + 1, + ) def test_projects_list(self): self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) def test_project_list_returns_projects_for_active_users_only(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice_user = alice_profile.user - shared_project = Project(name='demo2', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=alice_user, - organization=alice_user) + shared_project = Project( + name="demo2", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=alice_user, + organization=alice_user, + ) shared_project.save() # share project with self.user - shareProject = ShareProject( - shared_project, self.user.username, 'manager') + shareProject = ShareProject(shared_project, self.user.username, "manager") shareProject.save() # ensure when alice_user isn't active we can NOT # see the project she shared alice_user.is_active = False alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 1) - self.assertNotEqual(response.data[0].get( - 'projectid'), shared_project.id) + self.assertNotEqual(response.data[0].get("projectid"), shared_project.id) # ensure when alice_user is active we can # see the project she shared alice_user.is_active = True alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 2) shared_project_in_response = False for project in response.data: - if project.get('projectid') == shared_project.id: + if project.get("projectid") == shared_project.id: shared_project_in_response = True break self.assertTrue(shared_project_in_response) @@ -170,66 +183,58 @@ def test_project_list_returns_users_own_project_is_shared_to(self): contains all the users the project has been shared too """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - share_project = ShareProject( - self.project, 'alice', 'manager' - ) + share_project = ShareProject(self.project, "alice", "manager") share_project.save() # Ensure alice is in the list of users # When an owner requests for the project data - req = self.factory.get('/', **self.extra) + req = self.factory.get("/", **self.extra) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 2) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 2) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) # Ensure project managers can view all users the project was shared to - davis_data = {'username': 'davis', 'email': 'davis@localhost.com'} + davis_data = {"username": "davis", "email": "davis@localhost.com"} davis_profile = self._create_user_profile(davis_data) - dave_extras = { - 'HTTP_AUTHORIZATION': f'Token {davis_profile.user.auth_token}' - } + dave_extras = {"HTTP_AUTHORIZATION": f"Token {davis_profile.user.auth_token}"} share_project = ShareProject( - self.project, davis_profile.user.username, 'manager' + self.project, davis_profile.user.username, "manager" ) share_project.save() - req = self.factory.get('/', **dave_extras) + req = self.factory.get("/", **dave_extras) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 3) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 3) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) self.assertIn(self.user.username, shared_users) def test_projects_get(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - user_props = ['user', 'first_name', 'last_name', 'role', - 'is_org', 'metadata'] + user_props = ["user", "first_name", "last_name", "role", "is_org", "metadata"] user_props.sort() - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) # test serialized data - serializer = ProjectSerializer(self.project, - context={'request': request}) + serializer = ProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, serializer.data) self.assertIsNotNone(self.project_data) self.assertEqual(response.data, self.project_data) - res_user_props = list(response.data['users'][0]) + res_user_props = list(response.data["users"][0]) res_user_props.sort() self.assertEqual(res_user_props, user_props) @@ -240,105 +245,109 @@ def test_project_get_deleted_form(self): self.xform.deleted_at = self.xform.date_created self.xform.save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEqual(len(response.data.get('forms')), 0) + self.assertEqual(len(response.data.get("forms")), 0) self.assertEqual(response.status_code, 200) def test_none_empty_forms_and_dataview_properties_in_returned_json(self): self._publish_xls_form_to_project() self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertGreater(len(response.data.get('forms')), 0) - self.assertGreater( - len(response.data.get('data_views')), 0) - - form_obj_keys = list(response.data.get('forms')[0]) - data_view_obj_keys = list(response.data.get('data_views')[0]) - self.assertEqual(['date_created', - 'downloadable', - 'encrypted', - 'formid', - 'id_string', - 'is_merged_dataset', - 'last_submission_time', - 'last_updated_at', - 'name', - 'num_of_submissions', - 'published_by_formbuilder', - 'url'], - sorted(form_obj_keys)) - self.assertEqual(['columns', - 'dataviewid', - 'date_created', - 'date_modified', - 'instances_with_geopoints', - 'matches_parent', - 'name', - 'project', - 'query', - 'url', - 'xform'], - sorted(data_view_obj_keys)) + self.assertGreater(len(response.data.get("forms")), 0) + self.assertGreater(len(response.data.get("data_views")), 0) + + form_obj_keys = list(response.data.get("forms")[0]) + data_view_obj_keys = list(response.data.get("data_views")[0]) + self.assertEqual( + [ + "date_created", + "downloadable", + "encrypted", + "formid", + "id_string", + "is_merged_dataset", + "last_submission_time", + "last_updated_at", + "name", + "num_of_submissions", + "published_by_formbuilder", + "url", + ], + sorted(form_obj_keys), + ) + self.assertEqual( + [ + "columns", + "dataviewid", + "date_created", + "date_modified", + "instances_with_geopoints", + "matches_parent", + "name", + "project", + "query", + "url", + "xform", + ], + sorted(data_view_obj_keys), + ) self.assertEqual(response.status_code, 200) def test_projects_tags(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) - list_view = ProjectViewSet.as_view({ - 'get': 'list', - }) + view = ProjectViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) + list_view = ProjectViewSet.as_view( + { + "get": "list", + } + ) project_id = self.project.pk # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, [u'hello']) - self.assertEqual(get_latest_tags(self.project), [u'hello']) + self.assertEqual(response.data, ["hello"]) + self.assertEqual(get_latest_tags(self.project), ["hello"]) # check filter by tag - request = self.factory.get('/', data={"tags": "hello"}, **self.extra) + request = self.factory.get("/", data={"tags": "hello"}, **self.extra) self.project.refresh_from_db() request.user = self.user self.project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data[0], self.project_data) - request = self.factory.get('/', data={"tags": "goodbye"}, **self.extra) + request = self.factory.get("/", data={"tags": "goodbye"}, **self.extra) response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # remove tag "hello" - request = self.factory.delete('/', **self.extra) - response = view(request, pk=project_id, label='hello') + request = self.factory.delete("/", **self.extra) + response = view(request, pk=project_id, label="hello") self.assertEqual(response.status_code, 200) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) @@ -358,7 +367,7 @@ def test_project_create_other_account(self): # pylint: disable=C0103 Test that a user cannot create a project in a different user account without the right permission. """ - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) bob = self.user self._login_user_and_profile(alice_data) @@ -368,17 +377,23 @@ def test_project_create_other_account(self): # pylint: disable=C0103 } # Alice should not be able to create the form in bobs account. - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, { - 'owner': [u'You do not have permission to create a project in ' - 'the organization {}.'.format(bob)]}) + self.assertEqual( + response.data, + { + "owner": [ + "You do not have permission to create a project in " + "the organization {}.".format(bob) + ] + }, + ) self.assertEqual(Project.objects.count(), 0) # Give Alice the permission to create a project in Bob's account. ManagerRole.add(alice_profile.user, bob.profile) - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -397,25 +412,24 @@ def test_create_duplicate_project(self): """ # data to create project data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } # current number of projects count = Project.objects.count() # create project using data - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) after_count = Project.objects.count() @@ -423,28 +437,24 @@ def test_create_duplicate_project(self): # create another project using the same data request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, { - u'non_field_errors': - [u'The fields name, owner must make a unique set.'] - } + response.data, + {"non_field_errors": ["The fields name, owner must make a unique set."]}, ) final_count = Project.objects.count() self.assertEqual(after_count, final_count) def test_projects_create_no_metadata(self): data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "public": False, } - self._project_create(project_data=data, - merge=False) + self._project_create(project_data=data, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -457,7 +467,7 @@ def test_projects_create_no_metadata(self): def test_projects_create_many_users(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) self._project_create() projects = Project.objects.filter(created_by=self.user) @@ -469,235 +479,234 @@ def test_projects_create_many_users(self): def test_publish_xls_form_to_project(self): self._publish_xls_form_to_project() - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self._publish_xls_form_to_project() def test_num_datasets(self): self._publish_xls_form_to_project() self.project.refresh_from_db() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - self.assertEqual(self.project_data['num_datasets'], 1) + self.project, context={"request": request} + ).data + self.assertEqual(self.project_data["num_datasets"], 1) def test_last_submission_date(self): self._publish_xls_form_to_project() self._make_submissions() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project.refresh_from_db() self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - date_created = self.xform.instances.order_by( - '-date_created').values_list('date_created', flat=True)[0] - self.assertEqual(str(self.project_data['last_submission_date']), - str(date_created)) + self.project, context={"request": request} + ).data + date_created = self.xform.instances.order_by("-date_created").values_list( + "date_created", flat=True + )[0] + self.assertEqual( + str(self.project_data["last_submission_date"]), str(date_created) + ) def test_view_xls_form(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "forms"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) resultset = MetaData.objects.filter( - Q(object_id=self.xform.pk), Q(data_type='enketo_url') | - Q(data_type='enketo_preview_url') | - Q(data_type='enketo_single_submit_url')) - url = resultset.get(data_type='enketo_url') - preview_url = resultset.get(data_type='enketo_preview_url') - single_submit_url = resultset.get( - data_type='enketo_single_submit_url') - form_metadata = sorted([ + Q(object_id=self.xform.pk), + Q(data_type="enketo_url") + | Q(data_type="enketo_preview_url") + | Q(data_type="enketo_single_submit_url"), + ) + url = resultset.get(data_type="enketo_url") + preview_url = resultset.get(data_type="enketo_preview_url") + single_submit_url = resultset.get(data_type="enketo_single_submit_url") + form_metadata = sorted( + [ OrderedDict( [ - ('id', url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/::YY8M'), - ('data_type', 'enketo_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' - % url.pk), - ('date_created', url.date_created)]), + ("id", url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/::YY8M"), + ("data_type", "enketo_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ("url", "http://testserver/api/v1/metadata/%s" % url.pk), + ("date_created", url.date_created), + ] + ), OrderedDict( [ - ('id', preview_url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/preview/::YY8M'), - ('data_type', 'enketo_preview_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - preview_url.pk), - ('date_created', preview_url.date_created)]), + ("id", preview_url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/preview/::YY8M"), + ("data_type", "enketo_preview_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + "http://testserver/api/v1/metadata/%s" % preview_url.pk, + ), + ("date_created", preview_url.date_created), + ] + ), OrderedDict( [ - ('id', single_submit_url.pk), - ('xform', self.xform.pk), - ('data_value', - 'http://enketo.ona.io/single/::XZqoZ94y'), - ('data_type', 'enketo_single_submit_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - single_submit_url.pk), - ('date_created', single_submit_url.date_created)])], - key=itemgetter('id')) + ("id", single_submit_url.pk), + ("xform", self.xform.pk), + ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), + ("data_type", "enketo_single_submit_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + "http://testserver/api/v1/metadata/%s" + % single_submit_url.pk, + ), + ("date_created", single_submit_url.date_created), + ] + ), + ], + key=itemgetter("id"), + ) # test metadata content separately response_metadata = sorted( [dict(item) for item in response.data[0].pop("metadata")], - key=itemgetter('id')) + key=itemgetter("id"), + ) self.assertEqual(response_metadata, form_metadata) # remove metadata and date_modified - self.form_data.pop('metadata') - self.form_data.pop('date_modified') - self.form_data.pop('last_updated_at') - response.data[0].pop('date_modified') - response.data[0].pop('last_updated_at') - self.form_data.pop('has_id_string_changed') + self.form_data.pop("metadata") + self.form_data.pop("date_modified") + self.form_data.pop("last_updated_at") + response.data[0].pop("date_modified") + response.data[0].pop("last_updated_at") + self.form_data.pop("has_id_string_changed") self.assertDictEqual(dict(response.data[0]), dict(self.form_data)) def test_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertIn('forms', list(response.data)) - old_project_form_count = len(response.data['forms']) - old_project_num_datasets = response.data['num_datasets'] + self.assertIn("forms", list(response.data)) + old_project_form_count = len(response.data["forms"]) + old_project_num_datasets = response.data["num_datasets"] project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual(response.data['num_datasets'], 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["num_datasets"], 1) # Check if form transferred doesn't appear in the old project # details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual( - len(response.data['forms']), old_project_form_count - 1) - self.assertEqual( - response.data['num_datasets'], old_project_num_datasets - 1) + self.assertEqual(len(response.data["forms"]), old_project_form_count - 1) + self.assertEqual(response.data["num_datasets"], old_project_num_datasets - 1) def test_project_manager_can_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user as manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) def test_project_manager_can_assign_form_to_project_no_perm(self): # user must have owner/manager permissions - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user is not manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - self.assertFalse(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) ManagerRole.add(alice_profile.user, self.project) - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 403) def test_project_users_get_readonly_role_on_add_form(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) ReadOnlyRole.add(alice_profile.user, self.project) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) self._publish_xls_form_to_project() - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) - self.assertFalse(OwnerRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertFalse(OwnerRole.user_has_role(alice_profile.user, self.xform)) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_reject_form_transfer_if_target_account_has_id_string_already( - self, mock_send_mail): + self, mock_send_mail + ): # create bob's project and publish a form to it self._publish_xls_form_to_project() projectid = self.project.pk @@ -705,117 +714,119 @@ def test_reject_form_transfer_if_target_account_has_id_string_already( # create user alice alice_data = { - 'username': 'alice', - 'email': 'alice@localhost.com', - 'name': 'alice', - 'first_name': 'alice' + "username": "alice", + "email": "alice@localhost.com", + "name": "alice", + "first_name": "alice", } alice_profile = self._create_user_profile(alice_data) # share bob's project with alice - self.assertFalse( - ManagerRole.user_has_role(alice_profile.user, bobs_project)) - - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, bobs_project)) + + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.xform)) # log in as alice self._login_user_and_profile(extra_post_data=alice_data) # publish a form to alice's project that shares an id_string with # form published by bob - publish_data = {'owner': 'http://testserver/api/v1/users/alice'} + publish_data = {"owner": "http://testserver/api/v1/users/alice"} self._publish_xls_form_to_project(publish_data=publish_data) alices_form = XForm.objects.filter( - user__username='alice', id_string='transportation_2011_07_25')[0] + user__username="alice", id_string="transportation_2011_07_25" + )[0] alices_project = alices_form.project bobs_form = XForm.objects.filter( - user__username='bob', id_string='transportation_2011_07_25')[0] + user__username="bob", id_string="transportation_2011_07_25" + )[0] formid = bobs_form.id # try transfering bob's form from bob's project to alice's project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=alices_project.id) self.assertEqual(response.status_code, 400) self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) # try transfering bob's form from to alice's other project with # no forms - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) new_project_id = self.project.id - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=new_project_id) self.assertEqual(response.status_code, 400) self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_allow_form_transfer_if_org_is_owned_by_user( - self, mock_send_mail): + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_allow_form_transfer_if_org_is_owned_by_user(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() bobs_project = self.project - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # access bob's project initially to cache the forms list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) view(request, pk=bobs_project.pk) # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) org_project = self.project self.assertNotEqual(bobs_project.id, org_project.id) # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 201) # test that cached forms of a source project are cleared. Bob had one # forms initially and now it's been moved to the org project. - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=bobs_project.pk) bobs_results = response.data - self.assertListEqual(bobs_results.get('forms'), []) + self.assertListEqual(bobs_results.get("forms"), []) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() @@ -823,60 +834,66 @@ def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) # publish form to organization project self._publish_xls_form_to_project() # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_form_transfer_when_org_creator_creates_project( - self, mock_send_mail): + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -884,11 +901,13 @@ def test_form_transfer_when_org_creator_creates_project( self.assertIn(alice_profile.user, owners_team.user_set.all()) # let bob create a project in org - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, user_bob) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -897,29 +916,32 @@ def test_form_transfer_when_org_creator_creates_project( self._login_user_and_profile(alice_data) self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) @@ -931,34 +953,31 @@ def test_project_transfer_upgrades_permissions(self): bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) alice_url = f'http://testserver/api/v1/users/{alice_data["username"]}' - self._project_create({ - 'name': 'test project', - 'owner': alice_url, - 'public': False, - }, merge=False) + self._project_create( + { + "name": "test project", + "owner": alice_url, + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin - self._login_user_and_profile( - {'username': bob.username, 'email': bob.email}) + self._login_user_and_profile({"username": bob.username, "email": bob.email}) self._org_create() self.assertEqual(self.organization.created_by, bob) - org_url = ( - 'http://testserver/api/v1/users/' - f'{self.organization.user.username}') - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + org_url = "http://testserver/api/v1/users/" f"{self.organization.user.username}" + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -966,44 +985,37 @@ def test_project_transfer_upgrades_permissions(self): self.assertIn(alice_profile.user, owners_team.user_set.all()) # Share project to bob as editor - data = {'username': bob.username, 'role': EditorRole.name} - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + data = {"username": bob.username, "role": EditorRole.name} + view = ProjectViewSet.as_view({"post": "share"}) alice_auth_token = Token.objects.get(user=alice_profile.user).key - auth_credentials = { - 'HTTP_AUTHORIZATION': f'Token {alice_auth_token}' - } - request = self.factory.post('/', data=data, **auth_credentials) + auth_credentials = {"HTTP_AUTHORIZATION": f"Token {alice_auth_token}"} + request = self.factory.post("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 204) # Transfer project to Bobs Organization - data = {'owner': org_url, 'name': alice_project.name} - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) - request = self.factory.patch('/', data=data, **auth_credentials) + data = {"owner": org_url, "name": alice_project.name} + view = ProjectViewSet.as_view({"patch": "partial_update"}) + request = self.factory.patch("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) # Ensure all Admins have admin privileges to the project # once transferred - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) - project_users = response.data['users'] + project_users = response.data["users"] org_owners = get_or_create_organization_owners_team( - self.organization).user_set.all() + self.organization + ).user_set.all() for user in project_users: - owner = org_owners.filter(username=user['user']).first() + owner = org_owners.filter(username=user["user"]).first() if owner: - self.assertEqual(user['role'], OwnerRole.name) + self.assertEqual(user["role"], OwnerRole.name) @override_settings(ALLOW_PUBLIC_DATASETS=False) def test_disallow_public_project_creation(self): @@ -1011,55 +1023,55 @@ def test_disallow_public_project_creation(self): Test that an error is raised when a user tries to create a public project when public projects are disabled. """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': True + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "public": True, } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['public'][0], - "Public projects are currently disabled.") + response.data["public"][0], "Public projects are currently disabled." + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_form_transfer_when_org_admin_not_creator_creates_project( - self, mock_send_mail): + self, mock_send_mail + ): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -1068,11 +1080,13 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a project in org self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, alice_profile.user) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -1080,210 +1094,213 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a form in her personal project self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + role_class._remove_obj_permissions(alice_profile.user, self.project) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint_form_published_later(self, mock_send_mail): # create project self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) # publish form after project sharing self._publish_xls_form_to_project() - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + role_class._remove_obj_permissions(alice_profile.user, self.project) self.xform.delete() def test_project_share_remove_user(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) projectid = self.project.pk role_class = ReadOnlyRole - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) def test_project_filter_by_owner(self): """ Test projects endpoint filter by owner. """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com', - 'first_name': 'Alice', 'last_name': 'Alice'} + alice_data = { + "username": "alice", + "email": "alice@localhost.com", + "first_name": "Alice", + "last_name": "Alice", + } self._login_user_and_profile(alice_data) - ShareProject(self.project, self.user.username, 'readonly').save() + ShareProject(self.project, self.user.username, "readonly").save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = view(request, pk=self.project.pk) request.user = self.user self.project.refresh_from_db() bobs_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) # both bob's and alice's projects - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data - result = [{'owner': p.get('owner'), - 'projectid': p.get('projectid')} for p in response.data] - bob_data = {'owner': 'http://testserver/api/v1/users/bob', - 'projectid': bobs_project_data.get('projectid')} - alice_data = {'owner': 'http://testserver/api/v1/users/alice', - 'projectid': alice_project_data.get('projectid')} + self.project, context={"request": request} + ).data + result = [ + {"owner": p.get("owner"), "projectid": p.get("projectid")} + for p in response.data + ] + bob_data = { + "owner": "http://testserver/api/v1/users/bob", + "projectid": bobs_project_data.get("projectid"), + } + alice_data = { + "owner": "http://testserver/api/v1/users/alice", + "projectid": alice_project_data.get("projectid"), + } self.assertIn(bob_data, result) self.assertIn(alice_data, result) # only bob's project - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(bobs_project_data, response.data) self.assertNotIn(alice_project_data, response.data) # only alice's project - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) self.assertIn(alice_project_data, response.data) # none existent user - request = self.factory.get('/', {'owner': 'noone'}, **self.extra) + request = self.factory.get("/", {"owner": "noone"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # authenticated user can view public project - joe_data = {'username': 'joe', 'email': 'joe@localhost.com'} + joe_data = {"username": "joe", "email": "joe@localhost.com"} self._login_user_and_profile(joe_data) # should not show private projects when filtered by owner - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) @@ -1294,33 +1311,34 @@ def test_project_filter_by_owner(self): self.project.save() request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(alice_project_data, response.data) # should show deleted project public project when filtered by owner self.project.soft_delete() - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual([], response.data) def test_project_partial_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - metadata = '{"description": "Lorem ipsum",' \ - '"location": "Nakuru, Kenya",' \ - '"category": "water"' \ - '}' + metadata = ( + '{"description": "Lorem ipsum",' + '"location": "Nakuru, Kenya",' + '"category": "water"' + "}" + ) json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) @@ -1328,63 +1346,58 @@ def test_project_partial_updates(self): self.assertEqual(project.metadata, json_metadata) def test_cache_updated_on_project_update(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve', - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"get": "retrieve", "patch": "partial_update"}) self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(False, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) projectid = self.project.pk - data = {'public': True} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": True} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) def test_project_put_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'update' - }) + view = ProjectViewSet.as_view({"put": "update"}) projectid = self.project.pk data = { - 'name': u'updated name', - 'owner': 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'description', - 'location': 'Nairobi, Kenya', - 'category': 'health'} + "name": "updated name", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "description", + "location": "Nairobi, Kenya", + "category": "health", + }, } - data.update({'metadata': json.dumps(data.get('metadata'))}) - request = self.factory.put('/', data=data, **self.extra) + data.update({"metadata": json.dumps(data.get("metadata"))}) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=projectid) - data.update({'metadata': json.loads(data.get('metadata'))}) + data.update({"metadata": json.loads(data.get("metadata"))}) self.assertDictContainsSubset(data, response.data) def test_project_partial_updates_to_existing_metadata(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk metadata = '{"description": "Changed description"}' json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) json_metadata.update(project.metadata) @@ -1393,15 +1406,14 @@ def test_project_partial_updates_to_existing_metadata(self): def test_project_update_shared_cascades_to_xforms(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - data = {'public': 'true'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": "true"} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) - xforms_status = XForm.objects.filter(project__pk=projectid)\ - .values_list('shared', flat=True) + xforms_status = XForm.objects.filter(project__pk=projectid).values_list( + "shared", flat=True + ) self.assertTrue(xforms_status[0]) self.assertEqual(response.status_code, 200) @@ -1409,15 +1421,13 @@ def test_project_add_star(self): self._project_create() self.assertEqual(len(self.project.user_stars.all()), 0) - view = ProjectViewSet.as_view({ - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(response.status_code, 204) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) @@ -1426,37 +1436,30 @@ def test_create_project_invalid_metadata(self): Make sure that invalid metadata values are outright rejected Test fix for: https://github.com/onaio/onadata/issues/977 """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': "null", - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": "null", + "public": False, } request = self.factory.post( - '/', - data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) def test_project_delete_star(self): self._project_create() - view = ProjectViewSet.as_view({ - 'delete': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"delete": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() @@ -1467,70 +1470,67 @@ def test_project_get_starred_by(self): self._project_create() # add star as bob - view = ProjectViewSet.as_view({ - 'get': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"get": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # ensure email not shared user_profile_data = self.user_profile_data() - del user_profile_data['email'] - del user_profile_data['metadata'] + del user_profile_data["email"] + del user_profile_data["metadata"] - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) # add star as alice - request = self.factory.post('/', **self.extra) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # get star users as alice - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - alice_profile, bob_profile = sorted(response.data, - key=itemgetter('username')) - self.assertEquals(sorted(bob_profile.items()), - sorted(user_profile_data.items())) - self.assertEqual(alice_profile['username'], 'alice') + alice_profile, bob_profile = sorted(response.data, key=itemgetter("username")) + self.assertEquals( + sorted(bob_profile.items()), sorted(user_profile_data.items()) + ) + self.assertEqual(alice_profile["username"], "alice") def test_user_can_view_public_projects(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=public_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['public'], True) - self.assertEqual(response.data['projectid'], public_project.pk) - self.assertEqual(response.data['name'], 'demo') + self.assertEqual(response.data["public"], True) + self.assertEqual(response.data["projectid"], public_project.pk) + self.assertEqual(response.data["name"], "demo") def test_projects_same_name_diff_case(self): data1 = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - self._project_create(project_data=data1, - merge=False) + self._project_create(project_data=data1, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -1538,25 +1538,24 @@ def test_projects_same_name_diff_case(self): self.assertEqual(len(projects), 1) data2 = { - 'name': u'DEMO', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "DEMO", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data2), - content_type="application/json", **self.extra) + "/", data=json.dumps(data2), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) projects = Project.objects.all() self.assertEqual(len(projects), 1) @@ -1565,29 +1564,28 @@ def test_projects_same_name_diff_case(self): self.assertEqual(self.user, project.organization) def test_projects_get_exception(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) # does not exists response = view(request, pk=11111) self.assertEqual(response.status_code, 404) - self.assertEqual(response.data, {u'detail': u'Not found.'}) + self.assertEqual(response.data, {"detail": "Not found."}) # invalid id - response = view(request, pk='1w') + response = view(request, pk="1w") self.assertEqual(response.status_code, 400) - error_msg = ("Invalid value for project_id. It must be a " - "positive integer.") - self.assertEqual(str(response.data['detail']), error_msg) + error_msg = "Invalid value for project_id. It must be a " "positive integer." + self.assertEqual(str(response.data["detail"]), error_msg) def test_publish_to_public_project(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project @@ -1597,9 +1595,13 @@ def test_publish_to_public_project(self): self.assertEquals(self.xform.shared_data, True) def test_public_form_private_project(self): - self.project = Project(name='demo', shared=False, - metadata=json.dumps({'description': ''}), - created_by=self.user, organization=self.user) + self.project = Project( + name="demo", + shared=False, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) self.project.save() self._publish_xls_form_to_project() @@ -1642,28 +1644,30 @@ def test_public_form_private_project(self): self.assertFalse(self.project.shared) def test_publish_to_public_project_public_form(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project data = { - 'owner': 'http://testserver/api/v1/users/%s' + "owner": "http://testserver/api/v1/users/%s" % self.project.organization.username, - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) @@ -1672,78 +1676,66 @@ def test_publish_to_public_project_public_form(self): def test_project_all_users_can_share_remove_themselves(self): self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() - self.assertTrue(role_class.user_has_role(self.user, - self.project)) - self.assertTrue(role_class.user_has_role(self.user, - self.xform)) - data['role'] = role_name + self.assertTrue(role_class.user_has_role(self.user, self.project)) + self.assertTrue(role_class.user_has_role(self.user, self.xform)) + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_owner_cannot_remove_self_if_no_other_owner(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) ManagerRole.add(self.user, self.project) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} bob_profile = self._create_user_profile(tom_data) OwnerRole.add(bob_profile.user, self.project) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - error = {'remove': [u"Project requires at least one owner"]} + error = {"remove": ["Project requires at least one owner"]} self.assertEquals(response.data, error) - self.assertTrue(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertTrue(OwnerRole.user_has_role(bob_profile.user, self.project)) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} profile = self._create_user_profile(alice_data) OwnerRole.add(profile.user, self.project) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertFalse(OwnerRole.user_has_role(bob_profile.user, self.project)) def test_last_date_modified_changes_when_adding_new_form(self): self._project_create() @@ -1764,11 +1756,9 @@ def test_anon_project_form_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) + view = ProjectViewSet.as_view({"get": "forms"}) - request = self.factory.get('/') + request = self.factory.get("/") response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 404) @@ -1777,16 +1767,13 @@ def test_anon_project_list_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'list' - }) + view = ProjectViewSet.as_view({"get": "list"}) self.project.shared = True self.project.save() - public_projects = Project.objects.filter( - shared=True).count() + public_projects = Project.objects.filter(shared=True).count() - request = self.factory.get('/') + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) @@ -1795,45 +1782,42 @@ def test_anon_project_list_endpoint(self): def test_project_manager_can_delete_xform(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(ManagerRole.user_has_role(alice, self.project)) - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(ManagerRole.user_has_role(alice, self.project)) - self.assertTrue(alice.has_perm('delete_xform', self.xform)) + self.assertTrue(alice.has_perm("delete_xform", self.xform)) def test_move_project_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(OwnerRole.user_has_role(alice, self.project)) - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) - data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % alice.username - } - request = self.factory.patch('/', data=data_patch, **self.extra) + data_patch = {"owner": "http://testserver/api/v1/users/%s" % alice.username} + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) # bob cannot move project if he does not have can_add_project project @@ -1842,7 +1826,7 @@ def test_move_project_owner(self): # Give bob permission. ManagerRole.add(self.user, alice_profile) - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() @@ -1853,51 +1837,48 @@ def test_cannot_share_project_to_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - data = {'username': self.user.username, 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": self.user.username, + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['username'], [u"Cannot share project" - u" with the owner (bob)"]) + self.assertEqual( + response.data["username"], ["Cannot share project" " with the owner (bob)"] + ) self.assertTrue(OwnerRole.user_has_role(self.user, self.project)) def test_project_share_readonly(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) perms = role.get_object_users_with_permissions(self.project) for p in perms: - user = p.get('user') + user = p.get("user") if user == alice_profile.user: - r = p.get('role') + r = p.get("role") self.assertEquals(r, ReadOnlyRole.name) def test_move_project_owner_org(self): @@ -1907,19 +1888,17 @@ def test_move_project_owner_org(self): projectid = self.project.pk - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) old_org = self.project.organization data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % - self.organization.user.username + "owner": "http://testserver/api/v1/users/%s" + % self.organization.user.username } - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) - for a in response.data.get('teams'): - self.assertIsNotNone(a.get('role')) + for a in response.data.get("teams"): + self.assertIsNotNone(a.get("role")) self.assertEqual(response.status_code, 200) project = Project.objects.get(pk=projectid) @@ -1929,7 +1908,7 @@ def test_move_project_owner_org(self): def test_project_share_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) # set the user inactive @@ -1939,266 +1918,242 @@ def test_project_share_inactive_user(self): projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) self.assertEqual( response.data, - {'username': [u'The following user(s) is/are not active: alice']}) + {"username": ["The following user(s) is/are not active: alice"]}, + ) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_remove_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # set the user inactive self.assertTrue(alice_profile.user.is_active) alice_profile.user.is_active = False alice_profile.user.save() - data = {'username': 'alice', 'role': ReadOnlyRole.name, "remove": True} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name, "remove": True} + request = self.factory.put("/", data=data, **self.extra) self.assertEqual(response.status_code, 204) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_readonly_no_downloads(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_data = self._create_user_profile(tom_data) projectid = self.project.pk self.assertFalse( - ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, - self.project)) + ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, self.project) + ) - data = {'username': 'alice', 'role': ReadOnlyRoleNoDownload.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRoleNoDownload.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - data = {'username': 'tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') - elif user.get('user') == 'alice': - self.assertEquals(user.get('role'), 'readonly-no-download') - elif user.get('user') == 'tom': - self.assertEquals(user.get('role'), 'readonly') + if user.get("user") == "bob": + self.assertEquals(user.get("role"), "owner") + elif user.get("user") == "alice": + self.assertEquals(user.get("role"), "readonly-no-download") + elif user.get("user") == "tom": + self.assertEquals(user.get("role"), "readonly") def test_team_users_in_a_project(self): self._team_create() - project = Project.objects.create(name="Test Project", - organization=self.team.organization, - created_by=self.user, - metadata='{}') + project = Project.objects.create( + name="Test Project", + organization=self.team.organization, + created_by=self.user, + metadata="{}", + ) - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) user_chuck = chuck_profile.user - view = TeamViewSet.as_view({ - 'post': 'share'}) + view = TeamViewSet.as_view({"post": "share"}) - self.assertFalse(EditorRole.user_has_role(user_chuck, - project)) - data = {'role': EditorRole.name, - 'project': project.pk} + self.assertFalse(EditorRole.user_has_role(user_chuck, project)) + data = {"role": EditorRole.name, "project": project.pk} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, pk=self.team.pk) self.assertEqual(response.status_code, 204) tools.add_user_to_team(self.team, user_chuck) self.assertTrue(EditorRole.user_has_role(user_chuck, project)) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project.pk) - self.assertIsNotNone(response.data['teams']) - self.assertEquals(3, len(response.data['teams'])) - self.assertEquals(response.data['teams'][2]['role'], 'editor') - self.assertEquals(response.data['teams'][2]['users'][0], - str(chuck_profile.user.username)) + self.assertIsNotNone(response.data["teams"]) + self.assertEquals(3, len(response.data["teams"])) + self.assertEquals(response.data["teams"][2]["role"], "editor") + self.assertEquals( + response.data["teams"][2]["users"][0], str(chuck_profile.user.username) + ) def test_project_accesible_by_admin_created_by_diff_admin(self): self._org_create() # user 1 - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) # user 2 - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - }) + view = OrganizationProfileViewSet.as_view( + { + "post": "members", + } + ) # save the org creator bob = self.user data = json.dumps( - {"username": alice_profile.user.username, - "role": OwnerRole.name}) + {"username": alice_profile.user.username, "role": OwnerRole.name} + ) # create admin 1 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(201, response.status_code) data = json.dumps( - {"username": chuck_profile.user.username, - "role": OwnerRole.name}) + {"username": chuck_profile.user.username, "role": OwnerRole.name} + ) # create admin 2 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(201, response.status_code) # admin 2 creates a project self.user = chuck_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % - self.organization.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" + % self.organization.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } self._project_create(project_data=data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # admin 1 tries to access project created by admin 2 self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEquals(200, response.status_code) # assert admin can add colaborators - tompoo_data = {'username': 'tompoo', 'email': 'tompoo@localhost.com'} + tompoo_data = {"username": "tompoo", "email": "tompoo@localhost.com"} self._create_user_profile(tompoo_data) - data = {'username': 'tompoo', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "tompoo", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) self.user = bob - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % bob.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % bob.auth_token} # remove from admin org data = json.dumps({"username": alice_profile.user.username}) - view = OrganizationProfileViewSet.as_view({ - 'delete': 'members' - }) + view = OrganizationProfileViewSet.as_view({"delete": "members"}) request = self.factory.delete( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(200, response.status_code) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) @@ -2206,27 +2161,25 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): self.assertEquals(404, response.status_code) def test_public_project_on_creation(self): - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demopublic', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': True + "name": "demopublic", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": True, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) - project = Project.prefetched.filter( - name=data['name'], created_by=self.user)[0] + project = Project.prefetched.filter(name=data["name"], created_by=self.user)[0] self.assertTrue(project.shared) @@ -2235,109 +2188,111 @@ def test_permission_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project columns = json.dumps(self.xform.get_field_name_xpaths_only()) - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': columns, - 'query': '[ ]'} + data = { + "name": "My DataView", + "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "columns": columns, + "query": "[ ]", + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertTrue(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_permission_not_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]'} + data = { + "name": "My DataView", + "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]', + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertFalse(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_project_share_xform_meta_perms(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk @@ -2346,59 +2301,56 @@ def test_project_share_xform_meta_perms(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) for role_class in ROLES_ORDERED: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) if role_class in [EditorRole, EditorMinorRole]: self.assertFalse( - EditorRole.user_has_role(alice_profile.user, self.xform)) + EditorRole.user_has_role(alice_profile.user, self.xform) + ) self.assertTrue( - EditorMinorRole.user_has_role(alice_profile.user, - self.xform)) + EditorMinorRole.user_has_role(alice_profile.user, self.xform) + ) - elif role_class in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role_class in [DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole]: self.assertTrue( - DataEntryRole.user_has_role(alice_profile.user, - self.xform)) + DataEntryRole.user_has_role(alice_profile.user, self.xform) + ) else: self.assertTrue( - role_class.user_has_role(alice_profile.user, self.xform)) + role_class.user_has_role(alice_profile.user, self.xform) + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_atomicity(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk role_class = DataEntryOnlyRole - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) @@ -2407,11 +2359,14 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(role_class.user_has_role(alice, self.xform)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) mock_rm_xform_perms = MagicMock() - with patch('onadata.libs.models.share_project.remove_xform_permissions', mock_rm_xform_perms): # noqa + with patch( + "onadata.libs.models.share_project.remove_xform_permissions", + mock_rm_xform_perms, + ): # noqa mock_rm_xform_perms.side_effect = Exception() with self.assertRaises(Exception): response = view(request, pk=projectid) @@ -2420,7 +2375,7 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(mock_rm_xform_perms.called) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) # permissions have changed for both project and xform @@ -2429,71 +2384,67 @@ def test_project_share_atomicity(self, mock_send_mail): def test_project_list_by_owner(self): # create project and publish form to project - sluggie_data = {'username': 'sluggie', - 'email': 'sluggie@localhost.com'} + sluggie_data = {"username": "sluggie", "email": "sluggie@localhost.com"} self._login_user_and_profile(sluggie_data) self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share', - 'get': 'list' - }) + view = ProjectViewSet.as_view({"put": "share", "get": "list"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # Should list collaborators data = {"owner": "sluggie"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - users = response.data[0]['users'] + users = response.data[0]["users"] self.assertEqual(response.status_code, 200) - self.assertIn({'first_name': u'Bob', 'last_name': u'erama', - 'is_org': False, 'role': 'readonly', 'user': u'alice', - 'metadata': {}}, users) + self.assertIn( + { + "first_name": "Bob", + "last_name": "erama", + "is_org": False, + "role": "readonly", + "user": "alice", + "metadata": {}, + }, + users, + ) def test_projects_soft_delete(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'list', - 'delete': 'destroy' - }) + view = ProjectViewSet.as_view({"get": "list", "delete": "destroy"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) project_id = self.project.pk - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.user = self.user response = view(request, pk=project_id) self.assertEqual(response.status_code, 204) @@ -2501,14 +2452,14 @@ def test_projects_soft_delete(self): self.project = Project.objects.get(pk=project_id) self.assertIsNotNone(self.project.deleted_at) - self.assertTrue('deleted-at' in self.project.name) + self.assertTrue("deleted-at" in self.project.name) self.assertEqual(self.project.deleted_by, self.user) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertFalse(serializer.data in response.data) @@ -2518,45 +2469,40 @@ def test_project_share_multiple_users(self): Test that the project can be shared to multiple users """ self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice,tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') + if user.get("user") == "bob": + self.assertEquals(user.get("role"), "owner") else: - self.assertEquals(user.get('role'), 'readonly') + self.assertEquals(user.get("role"), "readonly") - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_sends_mail_on_multi_share(self, mock_send_mail): """ Test that on sharing a projects to multiple users mail is sent to all @@ -2564,109 +2510,96 @@ def test_sends_mail_on_multi_share(self, mock_send_mail): """ # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice,tom", + "role": ReadOnlyRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) self.assertEqual(mock_send_mail.call_count, 2) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) def test_project_caching(self): """ Test project viewset caching always keeps the latest version of the project in cache """ - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test on form detail update data returned from project viewset is # updated - form_view = XFormViewSet.as_view({ - 'patch': 'partial_update' - }) - post_data = {'title': 'new_name'} - request = self.factory.patch( - '/', data=post_data, **self.extra) + form_view = XFormViewSet.as_view({"patch": "partial_update"}) + post_data = {"title": "new_name"} + request = self.factory.patch("/", data=post_data, **self.extra) response = form_view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test that last_submission_time is updated correctly self._make_submissions() self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) - self.assertIsNotNone(response.data['forms'][0]['last_submission_time']) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) + self.assertIsNotNone(response.data["forms"][0]["last_submission_time"]) returned_date = dateutil.parser.parse( - response.data['forms'][0]['last_submission_time']) - self.assertEqual( - returned_date, - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"] + ) + self.assertEqual(returned_date, self.xform.time_of_last_submission()) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index 9b1f8f7f89..b525b3f523 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -2,7 +2,7 @@ import json import os from builtins import str -from future.moves.urllib.parse import urlparse, parse_qs +from six.moves.urllib.parse import urlparse, parse_qs from django.contrib.auth.models import User from django.core.cache import cache @@ -18,47 +18,45 @@ from registration.models import RegistrationProfile from rest_framework.authtoken.models import Token -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.apps.main.models.user_profile import \ - set_kpi_formbuilder_permissions +from onadata.apps.main.models.user_profile import set_kpi_formbuilder_permissions from onadata.libs.authentication import DigestAuthentication -from onadata.libs.serializers.user_profile_serializer import \ - _get_first_last_names +from onadata.libs.serializers.user_profile_serializer import _get_first_last_names def _profile_data(): return { - 'username': u'deno', - 'first_name': u'Dennis', - 'last_name': u'erama', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, - 'name': u'Dennis erama' + "username": "deno", + "first_name": "Dennis", + "last_name": "erama", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, + "name": "Dennis erama", } class TestUserProfileViewSet(TestAbstractViewSet): - def setUp(self): super(self.__class__, self).setUp() - self.view = UserProfileViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'patch': 'partial_update', - 'put': 'update' - }) + self.view = UserProfileViewSet.as_view( + { + "get": "list", + "post": "create", + "patch": "partial_update", + "put": "update", + } + ) def tearDown(self): """ @@ -68,47 +66,52 @@ def tearDown(self): cache.clear() def test_profiles_list(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) data = self.user_profile_data() - del data['metadata'] + del data["metadata"] self.assertEqual(response.data, [data]) def test_user_profile_list(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertEqual( - sorted([dict(d) for d in response.data], key=lambda x: x['id']), - sorted([user_profile_data, deno_profile_data], - key=lambda x: x['id'])) + sorted([dict(d) for d in response.data], key=lambda x: x["id"]), + sorted([user_profile_data, deno_profile_data], key=lambda x: x["id"]), + ) self.assertEqual(len(response.data), 2) # Inactive user not in list @@ -117,36 +120,39 @@ def test_user_profile_list(self): response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertNotIn(user_deno.pk, [user['id'] for user in response.data]) + self.assertNotIn(user_deno.pk, [user["id"] for user in response.data]) def test_user_profile_list_with_and_without_users_param(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # anonymous user gets empty response - request = self.factory.get('/') + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) # authenicated user without users query param only gets his/her profile - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertDictEqual(user_profile_data, response.data[0]) # authenicated user with blank users query param only gets his/her # profile data = {"users": ""} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -155,128 +161,131 @@ def test_user_profile_list_with_and_without_users_param(self): # authenicated user with comma separated usernames as users query param # value gets profiles of the usernames provided data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) self.assertEqual( - [dict(i) for i in response.data], - [user_profile_data, deno_profile_data] + [dict(i) for i in response.data], [user_profile_data, deno_profile_data] ) def test_profiles_get(self): """Test get user profile""" - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) + response.data, {"detail": "Expected URL keyword argument `user`."} + ) # by username - response = view(request, user='bob') - self.assertNotEqual(response.get('Cache-Control'), None) + response = view(request, user="bob") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) # by username mixed case - response = view(request, user='BoB') + response = view(request, user="BoB") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, self.user_profile_data()) # by pk response = view(request, user=self.user.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) def test_profiles_get_anon(self): - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) - request = self.factory.get('/') - response = view(request, user='bob') + response.data, {"detail": "Expected URL keyword argument `user`."} + ) + request = self.factory.get("/") + response = view(request, user="bob") data = self.user_profile_data() - del data['email'] - del data['metadata'] + del data["email"] + del data["metadata"] self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profiles_get_org_anon(self): self._org_create() self.client.logout() - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') - response = view(request, user=self.company_data['org']) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") + response = view(request, user=self.company_data["org"]) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], - self.company_data['name']) - self.assertIn('is_org', response.data) - self.assertEqual(response.data['is_org'], True) + self.assertEqual(response.data["first_name"], self.company_data["name"]) + self.assertIn("is_org", response.data) + self.assertEqual(response.data["is_org"], True) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_profile_create(self, mock_send_verification_email): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ('Dennis', 'erama') + password = data["password"] + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Dennis", "erama") self.assertEqual(response.data, data) self.assertTrue(mock_send_verification_email.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def _create_user_using_profiles_endpoint(self, data): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -286,20 +295,19 @@ def test_return_204_if_email_verification_variables_are_not_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email', - 'post': 'send_verification_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') + view = UserProfileViewSet.as_view( + {"get": "verify_email", "post": "send_verification_email"} ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data, **self.extra) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data, **self.extra) response = view(request) self.assertEquals(response.status_code, 204) - data = {'username': data.get('username')} - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + data = {"username": data.get("username")} + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) self.assertEquals(response.status_code, 204) @@ -308,81 +316,72 @@ def test_verification_key_is_valid(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username') - ) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEquals(response.data.get("username"), data.get("username")) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) def test_verification_key_is_valid_with_redirect_url_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) _data = { - 'verification_key': rp.activation_key, - 'redirect_url': 'http://red.ir.ect' + "verification_key": rp.activation_key, + "redirect_url": "http://red.ir.ect", } - request = self.factory.get('/', data=_data) + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 302) - self.assertIn('is_email_verified', response.url) - self.assertIn('username', response.url) + self.assertIn("is_email_verified", response.url) + self.assertIn("username", response.url) string_query_params = urlparse(response.url).query dict_query_params = parse_qs(string_query_params) - self.assertEquals(dict_query_params.get( - 'is_email_verified'), ['True']) - self.assertEquals( - dict_query_params.get('username'), - [data.get('username')] - ) + self.assertEquals(dict_query_params.get("is_email_verified"), ["True"]) + self.assertEquals(dict_query_params.get("username"), [data.get("username")]) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.apps.api.viewsets.user_profile_viewset.' - 'send_verification_email.delay') + ( + "onadata.apps.api.viewsets.user_profile_viewset." + "send_verification_email.delay" + ) ) - def test_sending_verification_email_succeeds( - self, mock_send_verification_email): + def test_sending_verification_email_succeeds(self, mock_send_verification_email): data = _profile_data() self._create_user_using_profiles_endpoint(data) - data = {'username': data.get('username')} - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + data = {"username": data.get("username")} + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) self.assertTrue(mock_send_verification_email.called) self.assertEquals(response.status_code, 200) self.assertEquals(response.data, "Verification email has been sent") - user = User.objects.get(username=data.get('username')) - self.assertFalse(user.profile.metadata.get('is_email_verified')) + user = User.objects.get(username=data.get("username")) + self.assertFalse(user.profile.metadata.get("is_email_verified")) @override_settings(VERIFIED_KEY_TEXT=None) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @@ -390,13 +389,11 @@ def test_sending_verification_email_fails(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) # trigger permission error when username of requesting user is # different from username in post details - request = self.factory.post('/', - data={'username': 'None'}, - **self.extra) + request = self.factory.post("/", data={"username": "None"}, **self.extra) response = view(request) self.assertEquals(response.status_code, 403) @@ -406,28 +403,34 @@ def test_profile_require_auth(self): Test profile require_auth is True when REQUIRE_ODK_AUTHENTICATION is set to True. """ - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - self.assertTrue(response.data.get('require_auth')) + self.assertTrue(response.data.get("require_auth")) def test_profile_create_without_last_name(self): data = { - 'username': u'deno', - 'first_name': u'Dennis', - 'email': u'deno@columbia.edu', + "username": "deno", + "first_name": "Dennis", + "email": "deno@columbia.edu", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -436,127 +439,136 @@ def test_disallow_profile_create_w_same_username(self): self._create_user_using_profiles_endpoint(data) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertTrue( - 'deno already exists' in response.data['username'][0]) + self.assertTrue("deno already exists" in response.data["username"][0]) def test_profile_create_with_malfunctioned_email(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = { - 'username': u'nguyenquynh', - 'first_name': u'Nguy\u1ec5n Th\u1ecb', - 'last_name': u'Di\u1ec5m Qu\u1ef3nh', - 'email': u'onademo0+nguyenquynh@gmail.com\ufeff', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'nguyenquynh.com', - 'twitter': u'nguyenquynh', - 'require_auth': False, - 'password': u'onademo', - 'is_org': False, + "username": "nguyenquynh", + "first_name": "Nguy\u1ec5n Th\u1ecb", + "last_name": "Di\u1ec5m Qu\u1ef3nh", + "email": "onademo0+nguyenquynh@gmail.com\ufeff", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "nguyenquynh.com", + "twitter": "nguyenquynh", + "require_auth": False, + "password": "onademo", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/nguyenquynh' - data['user'] = 'http://testserver/api/v1/users/nguyenquynh' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ( - u'Nguy\u1ec5n Th\u1ecb', u'Di\u1ec5m Qu\u1ef3nh') + password = data["password"] + del data["password"] + + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/nguyenquynh" + data["user"] = "http://testserver/api/v1/users/nguyenquynh" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Nguy\u1ec5n Th\u1ecb", "Di\u1ec5m Qu\u1ef3nh") self.assertEqual(response.data, data) - user = User.objects.get(username='nguyenquynh') + user = User.objects.get(username="nguyenquynh") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def test_profile_create_with_invalid_username(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - data['username'] = u'de' - del data['name'] + data["username"] = "de" + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('username'), - [u'Ensure this field has at least 3 characters.']) + response.data.get("username"), + ["Ensure this field has at least 3 characters."], + ) def test_profile_create_anon(self): data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json") + "/api/v1/profiles", data=json.dumps(data), content_type="application/json" + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - del data['email'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + del data["email"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profile_create_missing_name_field(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['first_name'] - del data['name'] + del data["first_name"] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) response.render() - self.assertContains(response, - 'Either name or first_name should be provided', - status_code=400) + self.assertContains( + response, "Either name or first_name should be provided", status_code=400 + ) def test_split_long_name_to_first_name_and_last_name(self): - name = "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression "\ + name = ( + "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression " "et de la Tolerance Dans La Region de" + ) first_name, last_name = _get_first_last_names(name) self.assertEqual(first_name, "(CPLTGL) Centre Pour la Promot") self.assertEqual(last_name, "ion de la Liberte D'Expression") def test_partial_updates(self): - self.assertEqual(self.user.profile.country, u'US') - country = u'KE' - username = 'george' - metadata = {u'computer': u'mac'} + self.assertEqual(self.user.profile.country, "US") + country = "KE" + username = "george" + metadata = {"computer": "mac"} json_metadata = json.dumps(metadata) - data = {'username': username, - 'country': country, - 'metadata': json_metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"username": username, "country": country, "metadata": json_metadata} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -568,13 +580,10 @@ def test_partial_updates_empty_metadata(self): profile = UserProfile.objects.get(user=self.user) profile.metadata = dict() profile.save() - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) - data = { - 'metadata': json_metadata, - 'overwrite': 'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": json_metadata, "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -582,505 +591,535 @@ def test_partial_updates_empty_metadata(self): def test_partial_updates_too_long(self): # the max field length for username is 30 in django - username = 'a' * 31 - data = {'username': username} - request = self.factory.patch('/', data=data, **self.extra) + username = "a" * 31 + data = {"username": username} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 400) self.assertEqual( response.data, - {'username': - [u'Ensure this field has no more than 30 characters.']}) + {"username": ["Ensure this field has no more than 30 characters."]}, + ) self.assertNotEqual(profile.user.username, username) def test_partial_update_metadata_field(self): - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) data = { - 'metadata': json_metadata, + "metadata": json_metadata, } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual(profile.metadata, metadata) # create a new key/value object if it doesn't exist - data = { - 'metadata': '{"zebra": {"key3": "value3"}}', - 'overwrite': u'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key3": "value3"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "value2", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "value2", "key3": "value3"}}, + ) # update an existing key/value object - data = { - 'metadata': '{"zebra": {"key2": "second"}}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key2": "second"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "second", "key3": "value3"}}, + ) # add a new key/value object if the key doesn't exist - data = { - 'metadata': '{"animal": "donkey"}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"animal": "donkey"}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, { - u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}, - u'animal': u'donkey'}) + profile.metadata, + { + "zebra": {"key1": "value1", "key2": "second", "key3": "value3"}, + "animal": "donkey", + }, + ) # don't pass overwrite param - data = {'metadata': '{"b": "caah"}'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}'} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) # pass 'overwrite' param whose value isn't false - data = {'metadata': '{"b": "caah"}', 'overwrite': u'falsey'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}', "overwrite": "falsey"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) def test_put_update(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # edit username with existing different user's username - data['username'] = 'bob' + data["username"] = "bob" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 400) # update - data['username'] = 'roger' - data['city'] = 'Nairobi' + data["username"] = "roger" + data["city"] = "Nairobi" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['city'], data['city']) + self.assertEqual(response.data["city"], data["city"]) def test_profile_create_mixed_case(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get( - user__username=data['username'].lower()) - data['id'] = profile.user.pk - data['gravatar'] = str(profile.gravatar) - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['username'] = u'deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"].lower()) + data["id"] = profile.user.pk + data["gravatar"] = str(profile.gravatar) + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["username"] = "deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - data['username'] = u'deno' - data['joined_on'] = str(profile.user.date_joined) + data["username"] = "deno" + data["joined_on"] = str(profile.user.date_joined) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertIn("%s already exists" % - data['username'], response.data['username']) + self.assertIn("%s already exists" % data["username"], response.data["username"]) def test_change_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "bobbob" new_password = "bobbob1" old_token = Token.objects.get(user=self.user).key - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") now = timezone.now().isoformat() user = User.objects.get(username__iexact=self.user.username) user_profile = UserProfile.objects.get(user_id=user.id) self.assertEqual(response.status_code, 200) self.assertEqual( - type(parse_datetime(user_profile.metadata['last_password_edit'])), - type(parse_datetime(now))) + type(parse_datetime(user_profile.metadata["last_password_edit"])), + type(parse_datetime(now)), + ) self.assertTrue(user.check_password(new_password)) - self.assertIn('access_token', response.data) - self.assertIn('temp_token', response.data) - self.assertEqual(response.data['username'], self.user.username) - self.assertNotEqual(response.data['access_token'], old_token) + self.assertIn("access_token", response.data) + self.assertIn("temp_token", response.data) + self.assertEqual(response.data["username"], self.user.username) + self.assertNotEqual(response.data["access_token"], old_token) # Assert requests made with the old tokens are rejected - post_data = { - 'current_password': new_password, - 'new_password': 'random'} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": new_password, "new_password": "random"} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 401) def test_change_password_wrong_current_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") user = User.objects.get(username__iexact=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual( - response.data, - "Invalid password. You have 9 attempts left.") + self.assertEqual(response.data, "Invalid password. You have 9 attempts left.") self.assertFalse(user.check_password(new_password)) def test_profile_create_with_name(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['first_name'] = 'Dennis' - data['last_name'] = 'deno' - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["first_name"] = "Dennis" + data["last_name"] = "deno" + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_twitter_username_validation(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - data['twitter'] = 'denoerama' + data["twitter"] = "denoerama" data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['twitter'], - [u'Invalid twitter username {}'.format(data['twitter'])] + response.data["twitter"], + ["Invalid twitter username {}".format(data["twitter"])], ) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_put_patch_method_on_names(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # update - data['first_name'] = 'Tom' - del data['name'] + data["first_name"] = "Tom" + del data["name"] request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) - response = self.view(request, user='deno') + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) - first_name = u'Henry' - last_name = u'Thierry' + first_name = "Henry" + last_name = "Thierry" - data = {'first_name': first_name, 'last_name': last_name} - request = self.factory.patch('/', data=data, **self.extra) + data = {"first_name": first_name, "last_name": last_name} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) - self.assertEqual(response.data['last_name'], data['last_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) + self.assertEqual(response.data["last_name"], data["last_name"]) - @patch('django.core.mail.EmailMultiAlternatives.send') + @patch("django.core.mail.EmailMultiAlternatives.send") def test_send_email_activation_api(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # Activation email not sent self.assertFalse(mock_send_mail.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_partial_update_without_password_fails(self): - data = {'email': 'user@example.com'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example.com"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - [u'Your password is required when updating your email address.'], - response.data) + ["Your password is required when updating your email address."], + response.data, + ) def test_partial_update_with_invalid_email_fails(self): - data = {'email': 'user@example'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_partial_update_email(self, mock_send_verification_email): profile_data = _profile_data() self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'user@example.com', - 'password': "invalid_password"} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": "invalid_password"} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) self.assertEqual(response.status_code, 400) - data = {'email': 'user@example.com', - 'password': profile_data.get('password')} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": profile_data.get("password")} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) profile = UserProfile.objects.get(user=rp.user) self.assertEqual(response.status_code, 200) - self.assertEqual(profile.user.email, 'user@example.com') + self.assertEqual(profile.user.email, "user@example.com") rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) def test_update_first_last_name_password_not_affected(self): - data = {'first_name': 'update_first', - 'last_name': 'update_last'} + data = {"first_name": "update_first", "last_name": "update_last"} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bobbob') + auth = DigestAuth("bob@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) - def test_partial_update_unique_email_api( - self, mock_send_verification_email): + def test_partial_update_unique_email_api(self, mock_send_verification_email): profile_data = { - 'username': u'bobby', - 'first_name': u'Bob', - 'last_name': u'Blender', - 'email': u'bobby@columbia.edu', - 'city': u'Bobville', - 'country': u'US', - 'organization': u'Bob Inc.', - 'website': u'bob.com', - 'twitter': u'boberama', - 'require_auth': False, - 'password': 'bobbob', - 'is_org': False, - 'name': u'Bob Blender' + "username": "bobby", + "first_name": "Bob", + "last_name": "Blender", + "email": "bobby@columbia.edu", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "website": "bob.com", + "twitter": "boberama", + "require_auth": False, + "password": "bobbob", + "is_org": False, + "name": "Bob Blender", } self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'example@gmail.com', - 'password': profile_data.get('password')} + data = {"email": "example@gmail.com", "password": profile_data.get("password")} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **deno_extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **deno_extra + ) response = self.view(request, user=rp.user.username) self.assertEqual(response.status_code, 200) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) - self.assertEqual(response.data['email'], data['email']) + self.assertEqual(response.data["email"], data["email"]) # create User request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") # Update email request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=user.username) self.assertEqual(response.status_code, 400) def test_profile_create_fails_with_long_first_and_last_names(self): data = { - 'username': u'machicimo', - 'email': u'mike@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'last_name': - u'undeomnisistenatuserrorsitvoluptatem', - 'first_name': - u'quirationevoluptatemsequinesciunt' + "username": "machicimo", + "email": "mike@columbia.edu", + "city": "Denoville", + "country": "US", + "last_name": "undeomnisistenatuserrorsitvoluptatem", + "first_name": "quirationevoluptatemsequinesciunt", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) - self.assertEqual(response.data['first_name'][0], - u'Ensure this field has no more than 30 characters.') - self.assertEqual(response.data['last_name'][0], - u'Ensure this field has no more than 30 characters.') + self.assertEqual( + response.data["first_name"][0], + "Ensure this field has no more than 30 characters.", + ) + self.assertEqual( + response.data["last_name"][0], + "Ensure this field has no more than 30 characters.", + ) self.assertEqual(response.status_code, 400) @all_requests def grant_perms_form_builder(self, url, request): - assert 'Authorization' in request.headers - assert request.headers.get('Authorization').startswith('Token') + assert "Authorization" in request.headers + assert request.headers.get("Authorization").startswith("Token") response = requests.Response() response.status_code = 201 - response._content = \ - { - "detail": "Successfully granted default model level perms to" - " user." - } + response._content = { + "detail": "Successfully granted default model level perms to" " user." + } return response def test_create_user_with_given_name(self): registered_functions = [r[1]() for r in signals.post_save.receivers] self.assertIn(set_kpi_formbuilder_permissions, registered_functions) with HTTMock(self.grant_perms_form_builder): - with self.settings(KPI_FORMBUILDER_URL='http://test_formbuilder$'): + with self.settings(KPI_FORMBUILDER_URL="http://test_formbuilder$"): extra_data = {"username": "rust"} self._login_user_and_profile(extra_post_data=extra_data) @@ -1088,110 +1127,137 @@ def test_get_monthly_submissions(self): """ Test getting monthly submissions for a user """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() count1 = Instance.objects.filter(xform=self.xform).count() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count1}) + self.assertEquals(response.data, {"private": count1}) # publish another form, make submission and make it public self._publish_form_with_hxl_support() - self.assertEquals(self.xform.id_string, 'hxl_example') - count2 = Instance.objects.filter(xform=self.xform).filter( - date_created__year=datetime.datetime.now().year).filter( - date_created__month=datetime.datetime.now().month).filter( - date_created__day=datetime.datetime.now().day).count() + self.assertEquals(self.xform.id_string, "hxl_example") + count2 = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=datetime.datetime.now().year) + .filter(date_created__month=datetime.datetime.now().month) + .filter(date_created__day=datetime.datetime.now().day) + .count() + ) self.xform.shared = True self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, {'private': count1, 'public': count2}) + self.assertEquals(response.data, {"private": count1, "public": count2}) def test_get_monthly_submissions_with_year_and_month_params(self): """ Test passing both month and year params """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make a submission dated 2013-02-18 self._publish_xls_form_to_project() survey = self.surveys[0] - submission_time = parse_datetime('2013-02-18 15:54:01Z') + submission_time = parse_datetime("2013-02-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=submission_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__month=2).filter(date_created__year=2013).count() + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=submission_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__month=2) + .filter(date_created__year=2013) + .count() + ) # get submission count and assert the response is correct - data = {'month': 2, 'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": 2, "year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) def test_monthly_submissions_with_month_param(self): """ Test that by passing only the value for month, the year is assumed to be the current year """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month year = datetime.datetime.now().year # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=year).filter(date_created__month=month).count() + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=year) + .filter(date_created__month=month) + .count() + ) - data = {'month': month} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": month} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) def test_monthly_submissions_with_year_param(self): """ Test that by passing only the value for year the month is assumed to be the current month """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month # publish form and make submissions dated the year 2013 # and the current month self._publish_xls_form_to_project() survey = self.surveys[0] - _time = parse_datetime('2013-' + str(month) + '-18 15:54:01Z') + _time = parse_datetime("2013-" + str(month) + "-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=2013).filter( - date_created__month=month).count() - - data = {'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=2013) + .filter(date_created__month=month) + .count() + ) + + data = {"year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) @override_settings(ENABLE_EMAIL_VERIFICATION=True) - @patch( - 'onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile') + @patch("onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile") def test_reads_from_master(self, mock_rp_class): """ Test that on failure to retrieve a UserProfile in the first @@ -1200,88 +1266,90 @@ def test_reads_from_master(self, mock_rp_class): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} mock_rp_class.DoesNotExist = RegistrationProfile.DoesNotExist mock_rp_class.objects.select_related( - 'user', 'user__profile' - ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] - request = self.factory.get('/', data=_data) + "user", "user__profile" + ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username')) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEquals(response.data.get("username"), data.get("username")) self.assertEqual( mock_rp_class.objects.select_related( - 'user', 'user__profile').get.call_count, 2) + "user", "user__profile" + ).get.call_count, + 2, + ) def test_change_password_attempts(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') - self.assertIsNone(cache.get('change_password_attempts-bob')) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") + self.assertIsNone(cache.get("change_password_attempts-bob")) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) # first attempt current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": current_password, "new_password": new_password} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password." - u" You have 9 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 1) + self.assertEqual( + response.data, "Invalid password." " You have 9 attempts left." + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 1) # second attempt - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password. You have 8 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 2) + self.assertEqual(response.data, "Invalid password. You have 8 attempts left.") + self.assertEqual(cache.get("change_password_attempts-bob"), 2) # check user is locked out - request = self.factory.post('/', data=post_data, **self.extra) - cache.set('change_password_attempts-bob', 9) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + cache.set("change_password_attempts-bob", 9) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Too many password reset attempts," - u" Try again in 30 minutes") - self.assertEqual(cache.get('change_password_attempts-bob'), 10) - self.assertIsNotNone(cache.get('lockout_change_password_user-bob')) + self.assertEqual( + response.data, + "Too many password reset attempts," " Try again in 30 minutes", + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 10) + self.assertIsNotNone(cache.get("lockout_change_password_user-bob")) lockout = datetime.datetime.strptime( - cache.get('lockout_change_password_user-bob'), '%Y-%m-%dT%H:%M:%S') + cache.get("lockout_change_password_user-bob"), "%Y-%m-%dT%H:%M:%S" + ) self.assertIsInstance(lockout, datetime.datetime) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") - @patch('onadata.apps.main.signals.send_generic_email') + @patch("onadata.apps.main.signals.send_generic_email") @override_settings(ENABLE_ACCOUNT_ACTIVATION_EMAILS=True) def test_account_activation_emails(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -1292,7 +1360,7 @@ def test_account_activation_emails(self, mock_send_mail): mock_send_mail.assert_has_calls( [ call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been " "successfully created! Kindly wait till an " "administrator activates your account." @@ -1300,14 +1368,14 @@ def test_account_activation_emails(self, mock_send_mail): "Please contact us with any questions, " "we're always happy to help." "\n\nThanks,\nThe Team at Ona", - "Ona account created - Pending activation" + "Ona account created - Pending activation", ), call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been activated." "\n\nThank you for choosing Ona!" "\n\nThanks,\nThe Team at Ona", - "Ona account activated" - ) + "Ona account activated", + ), ] ) diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index d6ed815c01..3fd81192de 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -21,7 +21,7 @@ from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ from django.utils.module_loading import import_string -from future.utils import listitems +from six import iteritems from guardian.shortcuts import assign_perm, get_perms_for_model, remove_perm from guardian.shortcuts import get_perms from kombu.exceptions import OperationalError @@ -31,7 +31,9 @@ from multidb.pinning import use_master from onadata.apps.api.models.organization_profile import ( - OrganizationProfile, create_owner_team_and_assign_permissions) + OrganizationProfile, + create_owner_team_and_assign_permissions, +) from onadata.apps.api.models.team import Team from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.forms import QuickConverter @@ -41,20 +43,41 @@ from onadata.libs.baseviewset import DefaultBaseViewset from onadata.libs.models.share_project import ShareProject from onadata.libs.permissions import ( - ROLES, DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, OwnerRole, get_role, - get_role_in_org, is_organization) + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + get_role, + get_role_in_org, + is_organization, +) from onadata.libs.utils.api_export_tools import custom_response_handler from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_OWNER_CACHE, PROJ_SUB_DATE_CACHE, reset_project_cache, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_SUB_DATE_CACHE, + reset_project_cache, + safe_delete, +) from onadata.libs.utils.common_tags import MEMBERS, XFORM_META_PERMS -from onadata.libs.utils.logger_tools import (publish_form, - response_with_mimetype_and_name) -from onadata.libs.utils.project_utils import (set_project_perms_to_xform, - set_project_perms_to_xform_async) -from onadata.libs.utils.user_auth import (check_and_set_form_by_id, - check_and_set_form_by_id_string) +from onadata.libs.utils.logger_tools import ( + publish_form, + response_with_mimetype_and_name, +) +from onadata.libs.utils.project_utils import ( + set_project_perms_to_xform, + set_project_perms_to_xform_async, +) +from onadata.libs.utils.user_auth import ( + check_and_set_form_by_id, + check_and_set_form_by_id_string, +) DECIMAL_PRECISION = 2 @@ -62,18 +85,21 @@ def _get_first_last_names(name): name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_id_for_type(record, mongo_field): date_field = datetime_from_str(record[mongo_field]) - mongo_str = '$' + mongo_field + mongo_str = "$" + mongo_field - return {"$substr": [mongo_str, 0, 10]} if isinstance(date_field, datetime)\ + return ( + {"$substr": [mongo_str, 0, 10]} + if isinstance(date_field, datetime) else mongo_str + ) def get_accessible_forms(owner=None, shared_form=False, shared_data=False): @@ -88,13 +114,14 @@ def get_accessible_forms(owner=None, shared_form=False, shared_data=False): if shared_form and not shared_data: xforms = xforms.filter(shared=True) - elif (shared_form and shared_data) or \ - (owner == 'public' and not shared_form and not shared_data): + elif (shared_form and shared_data) or ( + owner == "public" and not shared_form and not shared_data + ): xforms = xforms.filter(Q(shared=True) | Q(shared_data=True)) elif not shared_form and shared_data: xforms = xforms.filter(shared_data=True) - if owner != 'public': + if owner != "public": xforms = xforms.filter(user__username=owner) return xforms.distinct() @@ -109,33 +136,29 @@ def create_organization(name, creator): """ organization, _created = User.objects.get_or_create(username__iexact=name) organization_profile = OrganizationProfile.objects.create( - user=organization, creator=creator) + user=organization, creator=creator + ) return organization_profile def create_organization_object(org_name, creator, attrs=None): - '''Creates an OrganizationProfile object without saving to the database''' + """Creates an OrganizationProfile object without saving to the database""" attrs = attrs if attrs else {} - name = attrs.get('name', org_name) if attrs else org_name + name = attrs.get("name", org_name) if attrs else org_name first_name, last_name = _get_first_last_names(name) - email = attrs.get('email', u'') if attrs else u'' + email = attrs.get("email", "") if attrs else "" new_user = User( username=org_name, first_name=first_name, last_name=last_name, email=email, - is_active=getattr( - settings, - 'ORG_ON_CREATE_IS_ACTIVE', - True)) + is_active=getattr(settings, "ORG_ON_CREATE_IS_ACTIVE", True), + ) new_user.save() try: - registration_profile = RegistrationProfile.objects.create_profile( - new_user) + registration_profile = RegistrationProfile.objects.create_profile(new_user) except IntegrityError: - raise ValidationError(_( - u"%s already exists" % org_name - )) + raise ValidationError(_("%s already exists" % org_name)) if email: site = Site.objects.get(pk=settings.SITE_ID) registration_profile.send_activation_email(site) @@ -144,11 +167,12 @@ def create_organization_object(org_name, creator, attrs=None): name=name, creator=creator, created_by=creator, - city=attrs.get('city', u''), - country=attrs.get('country', u''), - organization=attrs.get('organization', u''), - home_page=attrs.get('home_page', u''), - twitter=attrs.get('twitter', u'')) + city=attrs.get("city", ""), + country=attrs.get("country", ""), + organization=attrs.get("organization", ""), + home_page=attrs.get("home_page", ""), + twitter=attrs.get("twitter", ""), + ) return profile @@ -157,15 +181,18 @@ def create_organization_team(organization, name, permission_names=None): Creates an organization team with the given permissions as defined in permission_names. """ - organization = organization.user \ - if isinstance(organization, OrganizationProfile) else organization + organization = ( + organization.user + if isinstance(organization, OrganizationProfile) + else organization + ) team = Team.objects.create(organization=organization, name=name) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") if permission_names: # get permission objects perms = Permission.objects.filter( - codename__in=permission_names, content_type=content_type) + codename__in=permission_names, content_type=content_type + ) if perms: team.permissions.add(*tuple(perms)) return team @@ -176,8 +203,7 @@ def get_organization_members_team(organization): create members team if it does not exist and add organization owner to the members team""" try: - team = Team.objects.get(name=u'%s#%s' % (organization.user.username, - MEMBERS)) + team = Team.objects.get(name="%s#%s" % (organization.user.username, MEMBERS)) except Team.DoesNotExist: team = create_organization_team(organization, MEMBERS) add_user_to_team(team, organization.user) @@ -192,14 +218,14 @@ def get_or_create_organization_owners_team(org): :param org: organization :return: Owners team of the organization """ - team_name = f'{org.user.username}#{Team.OWNER_TEAM_NAME}' + team_name = f"{org.user.username}#{Team.OWNER_TEAM_NAME}" try: team = Team.objects.get(name=team_name, organization=org.user) except Team.DoesNotExist: from multidb.pinning import use_master # pylint: disable=import-error + with use_master: - queryset = Team.objects.filter( - name=team_name, organization=org.user) + queryset = Team.objects.filter(name=team_name, organization=org.user) if queryset.count() > 0: return queryset.first() # pylint: disable=no-member return create_owner_team_and_assign_permissions(org) @@ -234,12 +260,11 @@ def remove_user_from_team(team, user): user.groups.remove(team) # remove the permission - remove_perm('view_team', user, team) + remove_perm("view_team", user, team) # if team is owners team remove more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: - owners_team = get_or_create_organization_owners_team( - team.organization.profile) + owners_team = get_or_create_organization_owners_team(team.organization.profile) members_team = get_organization_members_team(team.organization.profile) for perm in get_perms_for_model(Team): remove_perm(perm.codename, user, owners_team) @@ -260,7 +285,7 @@ def add_user_to_team(team, user): user.groups.add(team) # give the user perms to view the team - assign_perm('view_team', user, team) + assign_perm("view_team", user, team) # if team is owners team assign more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: @@ -293,10 +318,8 @@ def _get_owners(organization): return [ user - for user in get_or_create_organization_owners_team( - organization).user_set.all() - if get_role_in_org(user, organization) == 'owner' - and organization.user != user + for user in get_or_create_organization_owners_team(organization).user_set.all() + if get_role_in_org(user, organization) == "owner" and organization.user != user ] @@ -318,7 +341,8 @@ def create_organization_project(organization, project_name, created_by): name=project_name, organization=organization, created_by=created_by, - metadata='{}') + metadata="{}", + ) return project @@ -343,7 +367,8 @@ def publish_xlsform(request, owner, id_string=None, project=None): Publishes XLSForm & creates an XFormVersion object given a request. """ survey = do_publish_xlsform( - request.user, request.data, request.FILES, owner, id_string, project) + request.user, request.data, request.FILES, owner, id_string, project + ) return survey @@ -354,18 +379,20 @@ def do_publish_xlsform(user, post, files, owner, id_string=None, project=None): """ if id_string and project: xform = get_object_or_404( - XForm, user=owner, id_string=id_string, project=project) + XForm, user=owner, id_string=id_string, project=project + ) if not ManagerRole.user_has_role(user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - user, xform))) - elif not user.has_perm('can_add_xform', owner.profile): + _("{} has no manager/owner role to the form {}".format(user, xform)) + ) + elif not user.has_perm("can_add_xform", owner.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % { - 'user': user.username, - 'account': owner.username - })) + detail=_( + "User %(user)s has no permission to add xforms to " + "account %(account)s" + % {"user": user.username, "account": owner.username} + ) + ) def set_form(): """ @@ -373,8 +400,8 @@ def set_form(): """ if project: - args = (post and dict(listitems(post))) or {} - args['project'] = project.pk + args = (post and dict(list(iteritems(post)))) or {} + args["project"] = project.pk else: args = post @@ -395,11 +422,11 @@ def set_form(): Instantiates QuickConverter form to publish a form. """ props = { - 'project': project.pk, - 'dropbox_xls_url': request.data.get('dropbox_xls_url'), - 'xls_url': request.data.get('xls_url'), - 'csv_url': request.data.get('csv_url'), - 'text_xls_form': request.data.get('text_xls_form') + "project": project.pk, + "dropbox_xls_url": request.data.get("dropbox_xls_url"), + "xls_url": request.data.get("xls_url"), + "csv_url": request.data.get("csv_url"), + "text_xls_form": request.data.get("text_xls_form"), } form = QuickConverter(props, request.FILES) @@ -414,30 +441,32 @@ def id_string_exists_in_account(): otherwise returns False. """ try: - XForm.objects.get( - user=project.organization, id_string=xform.id_string) + XForm.objects.get(user=project.organization, id_string=xform.id_string) except XForm.DoesNotExist: return False return True - if 'formid' in request.data: - xform = get_object_or_404(XForm, pk=request.data.get('formid')) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, xform.project.pk)) + if "formid" in request.data: + xform = get_object_or_404(XForm, pk=request.data.get("formid")) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_FORMS_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, xform.project.pk)) if not ManagerRole.user_has_role(request.user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - request.user, xform))) - - msg = 'Form with the same id_string already exists in this account' + _( + "{} has no manager/owner role to the form {}".format( + request.user, xform + ) + ) + ) + + msg = "Form with the same id_string already exists in this account" # Without this check, a user can't transfer a form to projects that # he/she owns because `id_string_exists_in_account` will always # return true - if project.organization != xform.user and \ - id_string_exists_in_account(): + if project.organization != xform.user and id_string_exists_in_account(): raise exceptions.ParseError(_(msg)) xform.user = project.organization xform.project = project @@ -483,7 +512,8 @@ def get_xform(formid, request, username=None): if not xform: raise exceptions.PermissionDenied( - _("You do not have permission to view data from this form.")) + _("You do not have permission to view data from this form.") + ) return xform @@ -529,12 +559,13 @@ class TagForm(forms.Form): """ Simple TagForm class to validate tags in a request. """ + tags = TagField() form = TagForm(request.data) if form.is_valid(): - tags = form.cleaned_data.get('tags', None) + tags = form.cleaned_data.get("tags", None) if tags: for tag in tags: @@ -559,9 +590,9 @@ def get_data_value_objects(value): Looks for 'dataview 123 fruits.csv' or 'xform 345 fruits.csv'. """ model = None - if value.startswith('dataview'): + if value.startswith("dataview"): model = DataView - elif value.startswith('xform'): + elif value.startswith("xform"): model = XForm if model: @@ -575,8 +606,8 @@ def get_data_value_objects(value): if metadata.data_file: file_path = metadata.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") dfs = get_storage_class()() if dfs.exists(file_path): @@ -586,7 +617,8 @@ def get_data_value_objects(value): extension=extension, show_date=False, file_path=file_path, - full_mime=True) + full_mime=True, + ) return response return HttpResponseNotFound() @@ -600,10 +632,12 @@ def get_data_value_objects(value): return custom_response_handler( request, - xform, {}, + xform, + {}, Export.CSV_EXPORT, filename=filename, - dataview=dataview) + dataview=dataview, + ) return HttpResponseRedirect(metadata.data_value) @@ -615,7 +649,7 @@ def check_inherit_permission_from_project(xform_id, user): if there is a difference applies the project permissions to the user for the given xform_id. """ - if xform_id == 'public': + if xform_id == "public": return try: @@ -624,8 +658,12 @@ def check_inherit_permission_from_project(xform_id, user): return # get the project_xform - xform = XForm.objects.filter(pk=xform_id).select_related('project').only( - 'project_id', 'id').first() + xform = ( + XForm.objects.filter(pk=xform_id) + .select_related("project") + .only("project_id", "id") + .first() + ) if not xform: return @@ -656,8 +694,11 @@ def get_baseviewset_class(): the default in onadata :return: the default baseviewset """ - return import_string(settings.BASE_VIEWSET) \ - if settings.BASE_VIEWSET else DefaultBaseViewset + return ( + import_string(settings.BASE_VIEWSET) + if settings.BASE_VIEWSET + else DefaultBaseViewset + ) def generate_tmp_path(uploaded_csv_file): @@ -694,31 +735,31 @@ def get_xform_users(xform): org_members = get_team_members(user.username) data[user] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user in data: - data[perm.user]['permissions'].append(perm.permission.codename) + data[perm.user]["permissions"].append(perm.permission.codename) for user in org_members: if user not in data: data[user] = { - 'permissions': get_perms(user, xform), - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": get_perms(user, xform), + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } for k in data: - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], xform) - del data[k]['permissions'] + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], xform) + del data[k]["permissions"] return data @@ -731,8 +772,7 @@ def get_team_members(org_username): """ members = [] try: - team = Team.objects.get( - name="{}#{}".format(org_username, MEMBERS)) + team = Team.objects.get(name="{}#{}".format(org_username, MEMBERS)) except Team.DoesNotExist: pass else: @@ -750,20 +790,18 @@ def update_role_by_meta_xform_perms(xform): editor_role_list = [EditorRole, EditorMinorRole] editor_role = {role.name: role for role in editor_role_list} - dataentry_role_list = [ - DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole - ] + dataentry_role_list = [DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole] dataentry_role = {role.name: role for role in dataentry_role_list} if metadata: - meta_perms = metadata.data_value.split('|') + meta_perms = metadata.data_value.split("|") # update roles users = get_xform_users(xform) for user in users: - role = users.get(user).get('role') + role = users.get(user).get("role") if role in editor_role: role = ROLES.get(meta_perms[0]) role.add(user, xform) @@ -777,12 +815,13 @@ def replace_attachment_name_with_url(data): site_url = Site.objects.get_current().domain for record in data: - attachments: dict = record.json.get('_attachments') + attachments: dict = record.json.get("_attachments") if attachments: attachment_details = [ - (attachment['name'], attachment['download_url']) + (attachment["name"], attachment["download_url"]) for attachment in attachments - if 'download_url' in attachment] + if "download_url" in attachment + ] question_keys = list(record.json.keys()) question_values = list(record.json.values()) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 16db8e1c42..71f8b9a364 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -5,7 +5,7 @@ import datetime import json -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.conf import settings diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 644bc44c9b..5d3c3e5121 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -11,15 +11,19 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db import IntegrityError from django.db.models import Prefetch -from django.http import (HttpResponseBadRequest, HttpResponseForbidden, - HttpResponseRedirect, StreamingHttpResponse) +from django.http import ( + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseRedirect, + StreamingHttpResponse, +) from django.utils import six, timezone from django.utils.http import urlencode from django.utils.translation import ugettext as _ from django.views.decorators.cache import never_cache from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse try: from multidb.pinning import use_master @@ -46,65 +50,69 @@ from onadata.apps.logger.xform_instance_parser import XLSFormError from onadata.apps.viewer.models.export import Export from onadata.libs import authentication, filters -from onadata.libs.mixins.anonymous_user_public_forms_mixin import \ - AnonymousUserPublicFormsMixin -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.labels_mixin import LabelsMixin from onadata.libs.renderers import renderers -from onadata.libs.serializers.clone_xform_serializer import \ - CloneXFormSerializer -from onadata.libs.serializers.share_xform_serializer import \ - ShareXFormSerializer +from onadata.libs.serializers.clone_xform_serializer import CloneXFormSerializer +from onadata.libs.serializers.share_xform_serializer import ShareXFormSerializer from onadata.libs.serializers.xform_serializer import ( - XFormBaseSerializer, XFormCreateSerializer, XFormSerializer, - XFormVersionListSerializer) -from onadata.libs.utils.api_export_tools import (custom_response_handler, - get_async_response, - process_async_export, - response_for_format) + XFormBaseSerializer, + XFormCreateSerializer, + XFormSerializer, + XFormVersionListSerializer, +) +from onadata.libs.utils.api_export_tools import ( + custom_response_handler, + get_async_response, + process_async_export, + response_for_format, +) from onadata.libs.utils.common_tools import json_stream -from onadata.libs.utils.csv_import import (get_async_csv_submission_status, - submit_csv, submit_csv_async, - submission_xls_to_csv) +from onadata.libs.utils.csv_import import ( + get_async_csv_submission_status, + submit_csv, + submit_csv_async, + submission_xls_to_csv, +) from onadata.libs.utils.export_tools import parse_request_export_options from onadata.libs.utils.logger_tools import publish_form from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.string import str2bool -from onadata.libs.utils.viewer_tools import (get_enketo_urls, - generate_enketo_form_defaults, - get_form_url) +from onadata.libs.utils.viewer_tools import ( + get_enketo_urls, + generate_enketo_form_defaults, + get_form_url, +) from onadata.libs.exceptions import EnketoError from onadata.settings.common import XLS_EXTENSIONS, CSV_EXTENSION -from onadata.libs.utils.cache_tools import ( - PROJ_OWNER_CACHE, safe_delete) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete -ENKETO_AUTH_COOKIE = getattr(settings, 'ENKETO_AUTH_COOKIE', - '__enketo') -ENKETO_META_UID_COOKIE = getattr(settings, 'ENKETO_META_UID_COOKIE', - '__enketo_meta_uid') -ENKETO_META_USERNAME_COOKIE = getattr(settings, - 'ENKETO_META_USERNAME_COOKIE', - '__enketo_meta_username') +ENKETO_AUTH_COOKIE = getattr(settings, "ENKETO_AUTH_COOKIE", "__enketo") +ENKETO_META_UID_COOKIE = getattr( + settings, "ENKETO_META_UID_COOKIE", "__enketo_meta_uid" +) +ENKETO_META_USERNAME_COOKIE = getattr( + settings, "ENKETO_META_USERNAME_COOKIE", "__enketo_meta_username" +) BaseViewset = get_baseviewset_class() def upload_to_survey_draft(filename, username): - return os.path.join( - username, - 'survey-drafts', - os.path.split(filename)[1] - ) + return os.path.join(username, "survey-drafts", os.path.split(filename)[1]) def get_survey_dict(csv_name): - survey_file = default_storage.open(csv_name, 'rb') + survey_file = default_storage.open(csv_name, "rb") survey_dict = parse_file_to_json( - survey_file.name, default_name='data', file_object=survey_file) + survey_file.name, default_name="data", file_object=survey_file + ) return survey_dict @@ -116,14 +124,13 @@ def _get_user(username): def _get_owner(request): - owner = request.data.get('owner') or request.user + owner = request.data.get("owner") or request.user if isinstance(owner, six.string_types): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError( - u"User with username %s does not exist." % owner) + raise ValidationError("User with username %s does not exist." % owner) else: owner = owner_obj @@ -131,25 +138,26 @@ def _get_owner(request): def value_for_type(form, field, value): - if form._meta.get_field(field).get_internal_type() == 'BooleanField': + if form._meta.get_field(field).get_internal_type() == "BooleanField": return str2bool(value) return value def _try_update_xlsform(request, xform, owner): - survey = \ - utils.publish_xlsform(request, owner, xform.id_string, xform.project) + survey = utils.publish_xlsform(request, owner, xform.id_string, xform.project) if isinstance(survey, XForm): - serializer = XFormSerializer( - xform, context={'request': request}) + serializer = XFormSerializer(xform, context={"request": request}) # send form update notification send_message( - instance_id=xform.id, target_id=xform.id, - target_type=XFORM, user=request.user or owner, - message_verb=FORM_UPDATED) + instance_id=xform.id, + target_id=xform.id, + target_type=XFORM, + user=request.user or owner, + message_verb=FORM_UPDATED, + ) return Response(serializer.data, status=status.HTTP_200_OK) @@ -157,7 +165,7 @@ def _try_update_xlsform(request, xform, owner): def result_has_error(result): - return isinstance(result, dict) and result.get('type') + return isinstance(result, dict) and result.get("type") def get_survey_xml(csv_name): @@ -167,25 +175,23 @@ def get_survey_xml(csv_name): def set_enketo_signed_cookies(resp, username=None, json_web_token=None): - """Set signed cookies for JWT token in the HTTPResponse resp object. - """ + """Set signed cookies for JWT token in the HTTPResponse resp object.""" if not username and not json_web_token: return max_age = 30 * 24 * 60 * 60 * 1000 - enketo_meta_uid = {'max_age': max_age, 'salt': settings.ENKETO_API_SALT} - enketo = {'secure': False, 'salt': settings.ENKETO_API_SALT} + enketo_meta_uid = {"max_age": max_age, "salt": settings.ENKETO_API_SALT} + enketo = {"secure": False, "salt": settings.ENKETO_API_SALT} # add domain attribute if ENKETO_AUTH_COOKIE_DOMAIN is set in settings # i.e. don't add in development environment because cookie automatically # assigns 'localhost' as domain - if getattr(settings, 'ENKETO_AUTH_COOKIE_DOMAIN', None): - enketo_meta_uid['domain'] = settings.ENKETO_AUTH_COOKIE_DOMAIN - enketo['domain'] = settings.ENKETO_AUTH_COOKIE_DOMAIN + if getattr(settings, "ENKETO_AUTH_COOKIE_DOMAIN", None): + enketo_meta_uid["domain"] = settings.ENKETO_AUTH_COOKIE_DOMAIN + enketo["domain"] = settings.ENKETO_AUTH_COOKIE_DOMAIN resp.set_signed_cookie(ENKETO_META_UID_COOKIE, username, **enketo_meta_uid) - resp.set_signed_cookie(ENKETO_META_USERNAME_COOKIE, username, - **enketo_meta_uid) + resp.set_signed_cookie(ENKETO_META_USERNAME_COOKIE, username, **enketo_meta_uid) resp.set_signed_cookie(ENKETO_AUTH_COOKIE, json_web_token, **enketo) return resp @@ -202,18 +208,17 @@ def parse_webform_return_url(return_url, request): url = urlparse(return_url) try: # get jwt from url - probably zebra via enketo - jwt_param = [p for p in url.query.split('&') if p.startswith('jwt')] - jwt_param = jwt_param and jwt_param[0].split('=')[1] + jwt_param = [p for p in url.query.split("&") if p.startswith("jwt")] + jwt_param = jwt_param and jwt_param[0].split("=")[1] if not jwt_param: return except IndexError: pass - if '/_/' in return_url: # offline url - redirect_url = "%s://%s%s#%s" % ( - url.scheme, url.netloc, url.path, url.fragment) - elif '/::' in return_url: # non-offline url + if "/_/" in return_url: # offline url + redirect_url = "%s://%s%s#%s" % (url.scheme, url.netloc, url.path, url.fragment) + elif "/::" in return_url: # non-offline url redirect_url = "%s://%s%s" % (url.scheme, url.netloc, url.path) else: # unexpected format @@ -228,24 +233,27 @@ def parse_webform_return_url(return_url, request): if jwt_param: if request.user.is_anonymous: api_token = authentication.get_api_token(jwt_param) - if getattr(api_token, 'user'): + if getattr(api_token, "user"): username = api_token.user.username else: username = request.user.username response_redirect = set_enketo_signed_cookies( - response_redirect, username=username, json_web_token=jwt_param) + response_redirect, username=username, json_web_token=jwt_param + ) return response_redirect -class XFormViewSet(AnonymousUserPublicFormsMixin, - CacheControlMixin, - AuthenticateHeaderMixin, - ETagsMixin, - LabelsMixin, - BaseViewset, - ModelViewSet): +class XFormViewSet( + AnonymousUserPublicFormsMixin, + CacheControlMixin, + AuthenticateHeaderMixin, + ETagsMixin, + LabelsMixin, + BaseViewset, + ModelViewSet, +): """ Publish XLSForms, List, Retrieve Published Forms. """ @@ -259,46 +267,78 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, renderers.SurveyRenderer, renderers.OSMExportRenderer, renderers.ZipRenderer, - renderers.GoogleSheetsRenderer + renderers.GoogleSheetsRenderer, ] - queryset = XForm.objects.select_related('user', 'created_by')\ + queryset = ( + XForm.objects.select_related("user", "created_by") .prefetch_related( Prefetch( - 'xformuserobjectpermission_set', + "xformuserobjectpermission_set", queryset=XFormUserObjectPermission.objects.select_related( - 'user__profile__organizationprofile', - 'permission' - ) + "user__profile__organizationprofile", "permission" + ), ), - Prefetch('metadata_set'), - Prefetch('tags'), - Prefetch('dataview_set') - ).only( - 'id', 'id_string', 'title', 'shared', 'shared_data', - 'require_auth', 'created_by', 'num_of_submissions', - 'downloadable', 'encrypted', 'sms_id_string', - 'date_created', 'date_modified', 'last_submission_time', - 'uuid', 'bamboo_dataset', 'instances_with_osm', - 'instances_with_geopoints', 'version', 'has_hxl_support', - 'project', 'last_updated_at', 'user', 'allows_sms', 'description', - 'is_merged_dataset' + Prefetch("metadata_set"), + Prefetch("tags"), + Prefetch("dataview_set"), + ) + .only( + "id", + "id_string", + "title", + "shared", + "shared_data", + "require_auth", + "created_by", + "num_of_submissions", + "downloadable", + "encrypted", + "sms_id_string", + "date_created", + "date_modified", + "last_submission_time", + "uuid", + "bamboo_dataset", + "instances_with_osm", + "instances_with_geopoints", + "version", + "has_hxl_support", + "project", + "last_updated_at", + "user", + "allows_sms", + "description", + "is_merged_dataset", ) + ) serializer_class = XFormSerializer - lookup_field = 'pk' + lookup_field = "pk" extra_lookup_fields = None - permission_classes = [XFormPermissions, ] - updatable_fields = set(('description', 'downloadable', 'require_auth', - 'shared', 'shared_data', 'title')) - filter_backends = (filters.EnketoAnonDjangoObjectPermissionFilter, - filters.TagFilter, - filters.XFormOwnerFilter, - DjangoFilterBackend) - filter_fields = ('instances_with_osm',) + permission_classes = [ + XFormPermissions, + ] + updatable_fields = set( + ( + "description", + "downloadable", + "require_auth", + "shared", + "shared_data", + "title", + ) + ) + filter_backends = ( + filters.EnketoAnonDjangoObjectPermissionFilter, + filters.TagFilter, + filters.XFormOwnerFilter, + DjangoFilterBackend, + ) + filter_fields = ("instances_with_osm",) - public_forms_endpoint = 'public' + public_forms_endpoint = "public" def get_serializer_class(self): - if self.action == 'list': + if self.action == "list": return XFormBaseSerializer return super(XFormViewSet, self).get_serializer_class() @@ -307,37 +347,36 @@ def create(self, request, *args, **kwargs): try: owner = _get_owner(request) except ValidationError as e: - return Response({'message': e.messages[0]}, - status=status.HTTP_400_BAD_REQUEST) + return Response( + {"message": e.messages[0]}, status=status.HTTP_400_BAD_REQUEST + ) survey = utils.publish_xlsform(request, owner) if isinstance(survey, XForm): # survey is a DataDictionary we need an XForm to return the correct # role for the user after form publishing. - serializer = XFormCreateSerializer( - survey, context={'request': request}) + serializer = XFormCreateSerializer(survey, context={"request": request}) headers = self.get_success_headers(serializer.data) - return Response(serializer.data, status=status.HTTP_201_CREATED, - headers=headers) + return Response( + serializer.data, status=status.HTTP_201_CREATED, headers=headers + ) return Response(survey, status=status.HTTP_400_BAD_REQUEST) - @action(methods=['POST', 'GET'], detail=False) + @action(methods=["POST", "GET"], detail=False) def create_async(self, request, *args, **kwargs): - """ Temporary Endpoint for Async form creation """ + """Temporary Endpoint for Async form creation""" resp = headers = {} resp_code = status.HTTP_400_BAD_REQUEST - if request.method == 'GET': - self.etag_data = '{}'.format(timezone.now()) - survey = tasks.get_async_status( - request.query_params.get('job_uuid')) + if request.method == "GET": + self.etag_data = "{}".format(timezone.now()) + survey = tasks.get_async_status(request.query_params.get("job_uuid")) - if 'pk' in survey: - xform = XForm.objects.get(pk=survey.get('pk')) - serializer = XFormSerializer( - xform, context={'request': request}) + if "pk" in survey: + xform = XForm.objects.get(pk=survey.get("pk")) + serializer = XFormSerializer(xform, context={"request": request}) headers = self.get_success_headers(serializer.data) resp = serializer.data resp_code = status.HTTP_201_CREATED @@ -348,47 +387,51 @@ def create_async(self, request, *args, **kwargs): try: owner = _get_owner(request) except ValidationError as e: - return Response({'message': e.messages[0]}, - status=status.HTTP_400_BAD_REQUEST) + return Response( + {"message": e.messages[0]}, status=status.HTTP_400_BAD_REQUEST + ) - fname = request.FILES.get('xls_file').name - if isinstance(request.FILES.get('xls_file'), InMemoryUploadedFile): + fname = request.FILES.get("xls_file").name + if isinstance(request.FILES.get("xls_file"), InMemoryUploadedFile): xls_file_path = default_storage.save( - f'tmp/async-upload-{owner.username}-{fname}', - ContentFile(request.FILES.get('xls_file').read())) + f"tmp/async-upload-{owner.username}-{fname}", + ContentFile(request.FILES.get("xls_file").read()), + ) else: - xls_file_path = request.FILES.get( - 'xls_file').temporary_file_path() + xls_file_path = request.FILES.get("xls_file").temporary_file_path() resp.update( - {u'job_uuid': - tasks.publish_xlsform_async.delay( - request.user.id, - request.POST, - owner.id, - {'name': fname, 'path': xls_file_path}).task_id}) + { + "job_uuid": tasks.publish_xlsform_async.delay( + request.user.id, + request.POST, + owner.id, + {"name": fname, "path": xls_file_path}, + ).task_id + } + ) resp_code = status.HTTP_202_ACCEPTED return Response(data=resp, status=resp_code, headers=headers) - @action(methods=['GET', 'HEAD'], detail=True) + @action(methods=["GET", "HEAD"], detail=True) @never_cache - def form(self, request, format='json', **kwargs): + def form(self, request, format="json", **kwargs): form = self.get_object() - if format not in ['json', 'xml', 'xls', 'csv']: - return HttpResponseBadRequest('400 BAD REQUEST', - content_type='application/json', - status=400) - self.etag_data = '{}'.format(form.date_modified) + if format not in ["json", "xml", "xls", "csv"]: + return HttpResponseBadRequest( + "400 BAD REQUEST", content_type="application/json", status=400 + ) + self.etag_data = "{}".format(form.date_modified) filename = form.id_string + "." + format response = response_for_format(form, format=format) - response['Content-Disposition'] = 'attachment; filename=' + filename + response["Content-Disposition"] = "attachment; filename=" + filename return response - @action(methods=['GET'], detail=False) + @action(methods=["GET"], detail=False) def login(self, request, **kwargs): - return_url = request.query_params.get('return') + return_url = request.query_params.get("return") if return_url: redirect = parse_webform_return_url(return_url, request) @@ -396,86 +439,90 @@ def login(self, request, **kwargs): if redirect: return redirect - login_vars = {"login_url": settings.ENKETO_CLIENT_LOGIN_URL, - "return_url": urlencode({'return_url': return_url})} - client_login = '{login_url}?{return_url}'.format(**login_vars) + login_vars = { + "login_url": settings.ENKETO_CLIENT_LOGIN_URL, + "return_url": urlencode({"return_url": return_url}), + } + client_login = "{login_url}?{return_url}".format(**login_vars) return HttpResponseRedirect(client_login) - return HttpResponseForbidden( - "Authentication failure, cannot redirect") + return HttpResponseForbidden("Authentication failure, cannot redirect") - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def enketo(self, request, **kwargs): """Expose enketo urls.""" - survey_type = self.kwargs.get('survey_type') or \ - request.GET.get('survey_type') + survey_type = self.kwargs.get("survey_type") or request.GET.get("survey_type") self.object = self.get_object() form_url = get_form_url( - request, self.object.user.username, + request, + self.object.user.username, protocol=settings.ENKETO_PROTOCOL, - xform_pk=self.object.pk, generate_consistent_urls=True) + xform_pk=self.object.pk, + generate_consistent_urls=True, + ) - data = {'message': _(u"Enketo not properly configured.")} + data = {"message": _("Enketo not properly configured.")} http_status = status.HTTP_400_BAD_REQUEST try: # pass default arguments to enketo_url to prepopulate form fields request_vars = request.GET - defaults = generate_enketo_form_defaults( - self.object, **request_vars) - enketo_urls = get_enketo_urls( - form_url, self.object.id_string, **defaults) - offline_url = enketo_urls.get('offline_url') - preview_url = enketo_urls.get('preview_url') - single_submit_url = enketo_urls.get('single_url') + defaults = generate_enketo_form_defaults(self.object, **request_vars) + enketo_urls = get_enketo_urls(form_url, self.object.id_string, **defaults) + offline_url = enketo_urls.get("offline_url") + preview_url = enketo_urls.get("preview_url") + single_submit_url = enketo_urls.get("single_url") except EnketoError as e: - data = {'message': _(u"Enketo error: %s" % e)} + data = {"message": _("Enketo error: %s" % e)} else: - if survey_type == 'single': + if survey_type == "single": http_status = status.HTTP_200_OK data = {"single_submit_url": single_submit_url} else: http_status = status.HTTP_200_OK - data = {"enketo_url": offline_url, - "enketo_preview_url": preview_url, - "single_submit_url": single_submit_url} + data = { + "enketo_url": offline_url, + "enketo_preview_url": preview_url, + "single_submit_url": single_submit_url, + } return Response(data, http_status) - @action(methods=['POST', 'GET'], detail=False) + @action(methods=["POST", "GET"], detail=False) def survey_preview(self, request, **kwargs): username = request.user.username - if request.method.upper() == 'POST': + if request.method.upper() == "POST": if not username: raise ParseError("User has to be authenticated") - csv_data = request.data.get('body') + csv_data = request.data.get("body") if csv_data: - rand_name = "survey_draft_%s.csv" % ''.join( - random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) + rand_name = "survey_draft_%s.csv" % "".join( + random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) + ) csv_file = ContentFile(csv_data) csv_name = default_storage.save( - upload_to_survey_draft(rand_name, username), - csv_file) + upload_to_survey_draft(rand_name, username), csv_file + ) result = publish_form(lambda: get_survey_xml(csv_name)) if result_has_error(result): - raise ParseError(result.get('text')) + raise ParseError(result.get("text")) return Response( - {'unique_string': rand_name, 'username': username}, - status=200) + {"unique_string": rand_name, "username": username}, status=200 + ) else: - raise ParseError('Missing body') + raise ParseError("Missing body") - if request.method.upper() == 'GET': - filename = request.query_params.get('filename') - username = request.query_params.get('username') + if request.method.upper() == "GET": + filename = request.query_params.get("filename") + username = request.query_params.get("username") if not username: - raise ParseError('Username not provided') + raise ParseError("Username not provided") if not filename: raise ParseError("Filename MUST be provided") @@ -484,7 +531,7 @@ def survey_preview(self, request, **kwargs): result = publish_form(lambda: get_survey_xml(csv_name)) if result_has_error(result): - raise ParseError(result.get('text')) + raise ParseError(result.get("text")) self.etag_data = result @@ -506,87 +553,77 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) xform = self.get_object() - export_type = kwargs.get('format') or \ - request.query_params.get('format') + export_type = kwargs.get("format") or request.query_params.get("format") query = request.query_params.get("query") - token = request.GET.get('token') - meta = request.GET.get('meta') + token = request.GET.get("token") + meta = request.GET.get("meta") - if export_type is None or export_type in ['json', 'debug']: + if export_type is None or export_type in ["json", "debug"]: # perform default viewset retrieve, no data export return super(XFormViewSet, self).retrieve(request, *args, **kwargs) - return custom_response_handler(request, - xform, - query, - export_type, - token, - meta) + return custom_response_handler(request, xform, query, export_type, token, meta) - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def share(self, request, *args, **kwargs): self.object = self.get_object() - usernames_str = request.data.get("usernames", - request.data.get("username")) + usernames_str = request.data.get("usernames", request.data.get("username")) if not usernames_str: return Response(status=status.HTTP_400_BAD_REQUEST) role = request.data.get("role") # the serializer validates the role xform_id = self.object.pk - data_list = [{"xform": xform_id, - "username": username, - "role": role} - for username in usernames_str.split(",")] + data_list = [ + {"xform": xform_id, "username": username, "role": role} + for username in usernames_str.split(",") + ] serializer = ShareXFormSerializer(data=data_list, many=True) if serializer.is_valid(): serializer.save() else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def clone(self, request, *args, **kwargs): self.object = self.get_object() - data = {'xform': self.object.pk, - 'username': request.data.get('username')} - project = request.data.get('project_id') + data = {"xform": self.object.pk, "username": request.data.get("username")} + project = request.data.get("project_id") if project: - data['project'] = project + data["project"] = project serializer = CloneXFormSerializer(data=data) if serializer.is_valid(): - clone_to_user = User.objects.get(username=data['username']) - if not request.user.has_perm('can_add_xform', - clone_to_user.profile): + clone_to_user = User.objects.get(username=data["username"]) + if not request.user.has_perm("can_add_xform", clone_to_user.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add " - "xforms to account %(account)s" % - {'user': request.user.username, - 'account': data['username']})) + detail=_( + "User %(user)s has no permission to add " + "xforms to account %(account)s" + % {"user": request.user.username, "account": data["username"]} + ) + ) try: xform = serializer.save() except IntegrityError: raise ParseError( - 'A clone with the same id_string has already been created') + "A clone with the same id_string has already been created" + ) serializer = XFormSerializer( - xform.cloned_form, context={'request': request}) + xform.cloned_form, context={"request": request} + ) - return Response(data=serializer.data, - status=status.HTTP_201_CREATED) + return Response(data=serializer.data, status=status.HTTP_201_CREATED) - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) - @action( - methods=['POST', 'GET'], detail=True, - url_name='import', url_path='import') + @action(methods=["POST", "GET"], detail=True, url_name="import", url_path="import") def data_import(self, request, *args, **kwargs): - """ Endpoint for CSV and XLS data imports + """Endpoint for CSV and XLS data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and @@ -599,64 +636,75 @@ def data_import(self, request, *args, **kwargs): """ self.object = self.get_object() resp = {} - if request.method == 'GET': + if request.method == "GET": try: - resp.update(get_async_csv_submission_status( - request.query_params.get('job_uuid'))) + resp.update( + get_async_csv_submission_status( + request.query_params.get("job_uuid") + ) + ) self.last_modified_date = timezone.now() except ValueError: - raise ParseError(('The instance of the result is not a ' - 'basestring; the job_uuid variable might ' - 'be incorrect')) + raise ParseError( + ( + "The instance of the result is not a " + "basestring; the job_uuid variable might " + "be incorrect" + ) + ) else: - csv_file = request.FILES.get('csv_file', None) - xls_file = request.FILES.get('xls_file', None) + csv_file = request.FILES.get("csv_file", None) + xls_file = request.FILES.get("xls_file", None) if csv_file is None and xls_file is None: - resp.update({u'error': u'csv_file and xls_file field empty'}) + resp.update({"error": "csv_file and xls_file field empty"}) - elif xls_file and \ - xls_file.name.split('.')[-1] not in XLS_EXTENSIONS: - resp.update({u'error': u'xls_file not an excel file'}) + elif xls_file and xls_file.name.split(".")[-1] not in XLS_EXTENSIONS: + resp.update({"error": "xls_file not an excel file"}) - elif csv_file and csv_file.name.split('.')[-1] != CSV_EXTENSION: - resp.update({u'error': u'csv_file not a csv file'}) + elif csv_file and csv_file.name.split(".")[-1] != CSV_EXTENSION: + resp.update({"error": "csv_file not a csv file"}) else: - if xls_file and xls_file.name.split('.')[-1] in XLS_EXTENSIONS: + if xls_file and xls_file.name.split(".")[-1] in XLS_EXTENSIONS: csv_file = submission_xls_to_csv(xls_file) - overwrite = request.query_params.get('overwrite') - overwrite = True \ - if overwrite and overwrite.lower() == 'true' else False + overwrite = request.query_params.get("overwrite") + overwrite = True if overwrite and overwrite.lower() == "true" else False size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD try: csv_size = csv_file.size except AttributeError: csv_size = csv_file.__sizeof__() if csv_size < size_threshold: - resp.update(submit_csv(request.user.username, - self.object, csv_file, overwrite)) + resp.update( + submit_csv( + request.user.username, self.object, csv_file, overwrite + ) + ) else: csv_file.seek(0) - upload_to = os.path.join(request.user.username, - 'csv_imports', csv_file.name) + upload_to = os.path.join( + request.user.username, "csv_imports", csv_file.name + ) file_name = default_storage.save(upload_to, csv_file) - task = submit_csv_async.delay(request.user.username, - self.object.pk, file_name, - overwrite) + task = submit_csv_async.delay( + request.user.username, self.object.pk, file_name, overwrite + ) if task is None: - raise ParseError('Task not found') + raise ParseError("Task not found") else: - resp.update({u'task_id': task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, - status=status.HTTP_200_OK if resp.get('error') is None else - status.HTTP_400_BAD_REQUEST) + status=status.HTTP_200_OK + if resp.get("error") is None + else status.HTTP_400_BAD_REQUEST, + ) - @action(methods=['POST', 'GET'], detail=True) + @action(methods=["POST", "GET"], detail=True) def csv_import(self, request, *args, **kwargs): - """ Endpoint for CSV data imports + """Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and @@ -666,82 +714,94 @@ def csv_import(self, request, *args, **kwargs): """ self.object = self.get_object() resp = {} - if request.method == 'GET': + if request.method == "GET": try: - resp.update(get_async_csv_submission_status( - request.query_params.get('job_uuid'))) + resp.update( + get_async_csv_submission_status( + request.query_params.get("job_uuid") + ) + ) self.last_modified_date = timezone.now() except ValueError: - raise ParseError(('The instance of the result is not a ' - 'basestring; the job_uuid variable might ' - 'be incorrect')) + raise ParseError( + ( + "The instance of the result is not a " + "basestring; the job_uuid variable might " + "be incorrect" + ) + ) else: - csv_file = request.FILES.get('csv_file', None) + csv_file = request.FILES.get("csv_file", None) if csv_file is None: - resp.update({u'error': u'csv_file field empty'}) - elif csv_file.name.split('.')[-1] != CSV_EXTENSION: - resp.update({u'error': u'csv_file not a csv file'}) + resp.update({"error": "csv_file field empty"}) + elif csv_file.name.split(".")[-1] != CSV_EXTENSION: + resp.update({"error": "csv_file not a csv file"}) else: - overwrite = request.query_params.get('overwrite') - overwrite = True \ - if overwrite and overwrite.lower() == 'true' else False + overwrite = request.query_params.get("overwrite") + overwrite = True if overwrite and overwrite.lower() == "true" else False size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: - resp.update(submit_csv(request.user.username, - self.object, csv_file, overwrite)) + resp.update( + submit_csv( + request.user.username, self.object, csv_file, overwrite + ) + ) else: csv_file.seek(0) - upload_to = os.path.join(request.user.username, - 'csv_imports', csv_file.name) + upload_to = os.path.join( + request.user.username, "csv_imports", csv_file.name + ) file_name = default_storage.save(upload_to, csv_file) - task = submit_csv_async.delay(request.user.username, - self.object.pk, file_name, - overwrite) + task = submit_csv_async.delay( + request.user.username, self.object.pk, file_name, overwrite + ) if task is None: - raise ParseError('Task not found') + raise ParseError("Task not found") else: - resp.update({u'task_id': task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, - status=status.HTTP_200_OK if resp.get('error') is None else - status.HTTP_400_BAD_REQUEST) + status=status.HTTP_200_OK + if resp.get("error") is None + else status.HTTP_400_BAD_REQUEST, + ) def partial_update(self, request, *args, **kwargs): self.object = self.get_object() owner = self.object.user # updating the file - if request.FILES or set(['xls_url', - 'dropbox_xls_url', - 'text_xls_form']) & set(request.data): + if request.FILES or set(["xls_url", "dropbox_xls_url", "text_xls_form"]) & set( + request.data + ): return _try_update_xlsform(request, self.object, owner) try: - return super(XFormViewSet, self).partial_update(request, *args, - **kwargs) + return super(XFormViewSet, self).partial_update(request, *args, **kwargs) except XLSFormError as e: raise ParseError(str(e)) - @action(methods=['DELETE', 'GET'], detail=True) + @action(methods=["DELETE", "GET"], detail=True) def delete_async(self, request, *args, **kwargs): - if request.method == 'DELETE': + if request.method == "DELETE": xform = self.get_object() resp = { - u'job_uuid': tasks.delete_xform_async.delay( - xform.pk, - request.user.id).task_id, - u'time_async_triggered': datetime.now()} + "job_uuid": tasks.delete_xform_async.delay( + xform.pk, request.user.id + ).task_id, + "time_async_triggered": datetime.now(), + } # clear project from cache - safe_delete(f'{PROJ_OWNER_CACHE}{xform.project.pk}') + safe_delete(f"{PROJ_OWNER_CACHE}{xform.project.pk}") resp_code = status.HTTP_202_ACCEPTED - elif request.method == 'GET': - job_uuid = request.query_params.get('job_uuid') + elif request.method == "GET": + job_uuid = request.query_params.get("job_uuid") resp = tasks.get_async_status(job_uuid) resp_code = status.HTTP_202_ACCEPTED - self.etag_data = '{}'.format(timezone.now()) + self.etag_data = "{}".format(timezone.now()) return Response(data=resp, status=resp_code) @@ -752,51 +812,52 @@ def destroy(self, request, *args, **kwargs): return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def versions(self, request, *args, **kwargs): xform = self.get_object() - version_id = kwargs.get('version_id') - requested_format = kwargs.get('format') or 'json' + version_id = kwargs.get("version_id") + requested_format = kwargs.get("format") or "json" if not version_id: queryset = XFormVersion.objects.filter(xform=xform) serializer = XFormVersionListSerializer( - queryset, many=True, context={'request': request}) - return Response( - data=serializer.data, status=status.HTTP_200_OK) + queryset, many=True, context={"request": request} + ) + return Response(data=serializer.data, status=status.HTTP_200_OK) if version_id: - version = get_object_or_404( - XFormVersion, version=version_id, xform=xform) + version = get_object_or_404(XFormVersion, version=version_id, xform=xform) return response_for_format(version, format=requested_format) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): - job_uuid = request.query_params.get('job_uuid') - export_type = request.query_params.get('format') + job_uuid = request.query_params.get("job_uuid") + export_type = request.query_params.get("format") query = request.query_params.get("query") xform = self.get_object() - token = request.query_params.get('token') - meta = request.query_params.get('meta') - data_id = request.query_params.get('data_id') + token = request.query_params.get("token") + meta = request.query_params.get("meta") + data_id = request.query_params.get("data_id") options = parse_request_export_options(request.query_params) - options.update({ - 'meta': meta, - 'token': token, - 'data_id': data_id, - }) + options.update( + { + "meta": meta, + "token": token, + "data_id": data_id, + } + ) if query: - options.update({'query': query}) + options.update({"query": query}) - if request.query_params.get('format') in ['csvzip', 'savzip']: + if request.query_params.get("format") in ["csvzip", "savzip"]: # Overide renderer and mediatype because all response are # suppose to be in json # TODO: Avoid overiding the format query param for export type # DRF uses format to select the renderer self.request.accepted_renderer = renderers.JSONRenderer() - self.request.accepted_mediatype = 'application/json' + self.request.accepted_mediatype = "application/json" if job_uuid: try: @@ -812,19 +873,18 @@ def export_async(self, request, *args, **kwargs): resp = process_async_export(request, xform, export_type, options) if isinstance(resp, HttpResponseRedirect): - payload = { - "details": _("Google authorization needed"), - "url": resp.url - } - return Response(data=payload, - status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + payload = {"details": _("Google authorization needed"), "url": resp.url} + return Response( + data=payload, + status=status.HTTP_403_FORBIDDEN, + content_type="application/json", + ) - self.etag_data = '{}'.format(timezone.now()) + self.etag_data = "{}".format(timezone.now()) - return Response(data=resp, - status=status.HTTP_202_ACCEPTED, - content_type="application/json") + return Response( + data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" + ) def _get_streaming_response(self): """ @@ -836,18 +896,18 @@ def _get_streaming_response(self): queryset = queryset_iterator(self.object_list, chunksize=2000) def get_json_string(item): - return json.dumps(XFormBaseSerializer( - instance=item, - context={'request': self.request} - ).data) + return json.dumps( + XFormBaseSerializer( + instance=item, context={"request": self.request} + ).data + ) response = StreamingHttpResponse( - json_stream(queryset, get_json_string), - content_type="application/json" + json_stream(queryset, get_json_string), content_type="application/json" ) # calculate etag value and add it to response headers - if hasattr(self, 'etag_data'): + if hasattr(self, "etag_data"): self.set_etag_header(None, self.etag_data) self.set_cache_control(response) @@ -859,11 +919,12 @@ def get_json_string(item): return response def list(self, request, *args, **kwargs): - STREAM_DATA = getattr(settings, 'STREAM_DATA', False) + STREAM_DATA = getattr(settings, "STREAM_DATA", False) try: queryset = self.filter_queryset(self.get_queryset()) - last_modified = queryset.values_list('date_modified', flat=True)\ - .order_by('-date_modified') + last_modified = queryset.values_list("date_modified", flat=True).order_by( + "-date_modified" + ) if last_modified: self.etag_data = last_modified[0].isoformat() if STREAM_DATA: diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 0721fb46dd..af7e9c1c70 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -11,37 +11,47 @@ from django.db import connection from django.db.models.signals import post_delete, post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ +from six import python_2_unicode_compatible from onadata.apps.viewer.parsed_instance_tools import get_where_clause -from onadata.libs.models.sorting import (json_order_by, json_order_by_params, - sort_from_mongo_sort_str) -from onadata.libs.utils.cache_tools import (DATAVIEW_COUNT, - DATAVIEW_LAST_SUBMISSION_TIME, - XFORM_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.common_tags import (ATTACHMENTS, EDITED, GEOLOCATION, - ID, LAST_EDITED, MONGO_STRFTIME, - NOTES, SUBMISSION_TIME) - -SUPPORTED_FILTERS = ['=', '>', '<', '>=', '<=', '<>', '!='] -ATTACHMENT_TYPES = ['photo', 'audio', 'video'] -DEFAULT_COLUMNS = [ - ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] - - -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): - _json_str = u"json->>%s" +from onadata.libs.models.sorting import ( + json_order_by, + json_order_by_params, + sort_from_mongo_sort_str, +) +from onadata.libs.utils.cache_tools import ( + DATAVIEW_COUNT, + DATAVIEW_LAST_SUBMISSION_TIME, + XFORM_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + ATTACHMENTS, + EDITED, + GEOLOCATION, + ID, + LAST_EDITED, + MONGO_STRFTIME, + NOTES, + SUBMISSION_TIME, +) + +SUPPORTED_FILTERS = ["=", ">", "<", ">=", "<=", "<>", "!="] +ATTACHMENT_TYPES = ["photo", "audio", "video"] +DEFAULT_COLUMNS = [ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] + + +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): + _json_str = "json->>%s" if known_integers is not None and key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif known_dates is not None and key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif known_decimals is not None and key in known_decimals: - _json_str = u"CAST(JSON->>%s AS DECIMAL)" + _json_str = "CAST(JSON->>%s AS DECIMAL)" return _json_str @@ -51,10 +61,10 @@ def get_name_from_survey_element(element): def append_where_list(comp, t_list, json_str): - if comp in ['=', '>', '<', '>=', '<=']: - t_list.append(u"{} {} %s".format(json_str, comp)) - elif comp in ['<>', '!=']: - t_list.append(u"{} <> %s".format(json_str)) + if comp in ["=", ">", "<", ">=", "<="]: + t_list.append("{} {} %s".format(json_str, comp)) + elif comp in ["<>", "!="]: + t_list.append("{} <> %s".format(json_str)) return t_list @@ -63,8 +73,7 @@ def get_elements_of_type(xform, field_type): """ This function returns a list of column names of a specified type """ - return [f.get('name') - for f in xform.get_survey_elements_of_type(field_type)] + return [f.get("name") for f in xform.get_survey_elements_of_type(field_type)] def has_attachments_fields(data_view): @@ -95,8 +104,8 @@ class DataView(models.Model): """ name = models.CharField(max_length=255) - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) - project = models.ForeignKey('logger.Project', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) + project = models.ForeignKey("logger.Project", on_delete=models.CASCADE) columns = JSONField() query = JSONField(default=dict, blank=True) @@ -105,15 +114,19 @@ class DataView(models.Model): date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='dataview_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + related_name="dataview_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) class Meta: - app_label = 'logger' - verbose_name = _('Data View') - verbose_name_plural = _('Data Views') + app_label = "logger" + verbose_name = _("Data View") + verbose_name_plural = _("Data Views") def __str__(self): return getattr(self, "name", "") @@ -146,35 +159,39 @@ def _get_known_type(self, type_str): # pylint: disable=E1101 return [ get_name_from_survey_element(e) - for e in self.xform.get_survey_elements_of_type(type_str)] + for e in self.xform.get_survey_elements_of_type(type_str) + ] def get_known_integers(self): """Return elements of type integer""" - return self._get_known_type('integer') + return self._get_known_type("integer") def get_known_dates(self): """Return elements of type date""" - return self._get_known_type('date') + return self._get_known_type("date") def get_known_decimals(self): """Return elements of type decimal""" - return self._get_known_type('decimal') + return self._get_known_type("decimal") def has_instance(self, instance): """Return True if instance in set of dataview data""" cursor = connection.cursor() - sql = u"SELECT count(json) FROM logger_instance" - - where, where_params = self._get_where_clause(self, - self.get_known_integers(), - self.get_known_dates(), - self.get_known_decimals()) - sql_where = u"" + sql = "SELECT count(json) FROM logger_instance" + + where, where_params = self._get_where_clause( + self, + self.get_known_integers(), + self.get_known_dates(), + self.get_known_decimals(), + ) + sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE xform_id = %s AND id = %s" + sql_where \ - + u" AND deleted_at IS NULL" + sql += ( + " WHERE xform_id = %s AND id = %s" + sql_where + " AND deleted_at IS NULL" + ) # pylint: disable=E1101 params = [self.xform.pk, instance.id] + where_params @@ -192,20 +209,25 @@ def soft_delete(self, user=None): uniqueness constraint. """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix - update_fields = ['date_modified', 'deleted_at', 'name', 'deleted_by'] + update_fields = ["date_modified", "deleted_at", "name", "deleted_by"] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) @classmethod - def _get_where_clause(cls, data_view, form_integer_fields=[], - form_date_fields=[], form_decimal_fields=[]): - known_integers = ['_id'] + form_integer_fields - known_dates = ['_submission_time'] + form_date_fields + def _get_where_clause( + cls, + data_view, + form_integer_fields=[], + form_date_fields=[], + form_decimal_fields=[], + ): + known_integers = ["_id"] + form_integer_fields + known_dates = ["_submission_time"] + form_date_fields known_decimals = form_decimal_fields where = [] where_params = [] @@ -216,19 +238,19 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], or_params = [] for qu in query: - comp = qu.get('filter') - column = qu.get('column') - value = qu.get('value') - condi = qu.get('condition') + comp = qu.get("filter") + column = qu.get("column") + value = qu.get("value") + condi = qu.get("condition") - json_str = _json_sql_str(column, known_integers, known_dates, - known_decimals) + json_str = _json_sql_str( + column, known_integers, known_dates, known_decimals + ) if comp in known_dates: - value = datetime.datetime.strptime( - value[:19], MONGO_STRFTIME) + value = datetime.datetime.strptime(value[:19], MONGO_STRFTIME) - if condi and condi.lower() == 'or': + if condi and condi.lower() == "or": or_where = append_where_list(comp, or_where, json_str) or_params.extend((column, text(value))) else: @@ -236,7 +258,7 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], where_params.extend((column, text(value))) if or_where: - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] where += or_where where_params.extend(or_params) @@ -246,19 +268,18 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], @classmethod def query_iterator(cls, sql, fields=None, params=[], count=False): cursor = connection.cursor() - sql_params = tuple( - i if isinstance(i, tuple) else text(i) for i in params) + sql_params = tuple(i if isinstance(i, tuple) else text(i) for i in params) if count: - from_pos = sql.upper().find(' FROM') + from_pos = sql.upper().find(" FROM") if from_pos != -1: - sql = u"SELECT COUNT(*) " + sql[from_pos:] + sql = "SELECT COUNT(*) " + sql[from_pos:] - order_pos = sql.upper().find('ORDER BY') + order_pos = sql.upper().find("ORDER BY") if order_pos != -1: sql = sql[:order_pos] - fields = [u'count'] + fields = ["count"] cursor.execute(sql, sql_params) @@ -274,16 +295,22 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): yield dict(zip(fields, [row[0].get(f) for f in fields])) @classmethod - def generate_query_string(cls, data_view, start_index, limit, - last_submission_time, all_data, sort, - filter_query=None): - additional_columns = [GEOLOCATION] \ - if data_view.instances_with_geopoints else [] + def generate_query_string( + cls, + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query=None, + ): + additional_columns = [GEOLOCATION] if data_view.instances_with_geopoints else [] if has_attachments_fields(data_view): additional_columns += [ATTACHMENTS] - sql = u"SELECT json FROM logger_instance" + sql = "SELECT json FROM logger_instance" if all_data or data_view.matches_parent: columns = None elif last_submission_time: @@ -300,12 +327,15 @@ def generate_query_string(cls, data_view, start_index, limit, data_view, data_view.get_known_integers(), data_view.get_known_dates(), - data_view.get_known_decimals()) + data_view.get_known_decimals(), + ) if filter_query: - add_where, add_where_params = \ - get_where_clause(filter_query, data_view.get_known_integers(), - data_view.get_known_decimals()) + add_where, add_where_params = get_where_clause( + filter_query, + data_view.get_known_integers(), + data_view.get_known_decimals(), + ) if add_where: where = where + add_where @@ -313,55 +343,74 @@ def generate_query_string(cls, data_view, start_index, limit, sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) if data_view.xform.is_merged_dataset: - sql += u" WHERE xform_id IN %s " + sql_where \ - + u" AND deleted_at IS NULL" - params = [tuple(list( - data_view.xform.mergedxform.xforms.values_list('pk', flat=True) - ))] + where_params + sql += " WHERE xform_id IN %s " + sql_where + " AND deleted_at IS NULL" + params = [ + tuple( + list( + data_view.xform.mergedxform.xforms.values_list("pk", flat=True) + ) + ) + ] + where_params else: - sql += u" WHERE xform_id = %s " + sql_where \ - + u" AND deleted_at IS NULL" + sql += " WHERE xform_id = %s " + sql_where + " AND deleted_at IS NULL" params = [data_view.xform.pk] + where_params if sort is not None: - sort = ['id'] if sort is None\ - else sort_from_mongo_sort_str(sort) - sql = u"{} {}".format(sql, json_order_by(sort)) + sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) + sql = "{} {}".format(sql, json_order_by(sort)) params = params + json_order_by_params(sort) elif last_submission_time is False: - sql += ' ORDER BY id' + sql += " ORDER BY id" if start_index is not None: - sql += u" OFFSET %s" + sql += " OFFSET %s" params += [start_index] if limit is not None: - sql += u" LIMIT %s" + sql += " LIMIT %s" params += [limit] if last_submission_time: - sql += u" ORDER BY date_created DESC" - sql += u" LIMIT 1" + sql += " ORDER BY date_created DESC" + sql += " LIMIT 1" - return (sql, columns, params, ) + return ( + sql, + columns, + params, + ) @classmethod - def query_data(cls, data_view, start_index=None, limit=None, count=None, - last_submission_time=False, all_data=False, sort=None, - filter_query=None): + def query_data( + cls, + data_view, + start_index=None, + limit=None, + count=None, + last_submission_time=False, + all_data=False, + sort=None, + filter_query=None, + ): (sql, columns, params) = cls.generate_query_string( - data_view, start_index, limit, last_submission_time, - all_data, sort, filter_query) + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query, + ) try: - records = [record for record in DataView.query_iterator(sql, - columns, - params, - count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, count) + ] except Exception as e: return {"error": _(text(e))} @@ -369,23 +418,18 @@ def query_data(cls, data_view, start_index=None, limit=None, count=None, def clear_cache(sender, instance, **kwargs): - """ Post delete handler for clearing the dataview cache. - """ - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + """Post delete handler for clearing the dataview cache.""" + safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) def clear_dataview_cache(sender, instance, **kwargs): - """ Post Save handler for clearing dataview cache on serialized fields. - """ - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, instance.project.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform.pk)) - safe_delete( - '{}{}'.format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + """Post Save handler for clearing dataview cache on serialized fields.""" + safe_delete("{}{}".format(PROJ_OWNER_CACHE, instance.project.pk)) + safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform.pk)) + safe_delete("{}{}".format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) + safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) -post_save.connect(clear_dataview_cache, sender=DataView, - dispatch_uid='clear_cache') +post_save.connect(clear_dataview_cache, sender=DataView, dispatch_uid="clear_cache") -post_delete.connect(clear_cache, sender=DataView, - dispatch_uid='clear_xform_cache') +post_delete.connect(clear_cache, sender=DataView, dispatch_uid="clear_xform_cache") diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index fdd90df0b9..721fa7ff29 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -19,7 +19,7 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import ugettext as _ -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview diff --git a/onadata/apps/logger/models/open_data.py b/onadata/apps/logger/models/open_data.py index 742393c3a2..c7c85f852d 100644 --- a/onadata/apps/logger/models/open_data.py +++ b/onadata/apps/logger/models/open_data.py @@ -7,7 +7,7 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from onadata.libs.utils.common_tools import get_uuid @@ -18,11 +18,12 @@ class OpenData(models.Model): OpenData model represents a way to access private datasets without authentication using the unique uuid. """ + name = models.CharField(max_length=255) uuid = models.CharField(max_length=32, default=get_uuid, unique=True) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") active = models.BooleanField(default=True) date_created = models.DateTimeField(auto_now_add=True) @@ -32,7 +33,7 @@ def __str__(self): return getattr(self, "name", "") class Meta: - app_label = 'logger' + app_label = "logger" def get_or_create_opendata(xform): @@ -47,8 +48,8 @@ def get_or_create_opendata(xform): return OpenData.objects.get_or_create( object_id=xform.id, defaults={ - 'name': xform.id_string, - 'content_type': content_type, - 'content_object': xform, - } + "name": xform.id_string, + "content_type": content_type, + "content_object": xform, + }, ) diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index 6865da516d..81c03b1bd1 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -10,7 +10,7 @@ from django.db.models import Prefetch from django.db.models.signals import post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -24,38 +24,59 @@ class PrefetchManager(models.Manager): def get_queryset(self): from onadata.apps.logger.models.xform import XForm from onadata.apps.api.models.team import Team - return super(PrefetchManager, self).get_queryset().select_related( - 'created_by', 'organization' - ).prefetch_related( - Prefetch('xform_set', - queryset=XForm.objects.filter(deleted_at__isnull=True) - .select_related('user') - .prefetch_related('user') - .prefetch_related('dataview_set') - .prefetch_related('metadata_set') - .only('id', 'user', 'project', 'title', 'date_created', - 'last_submission_time', 'num_of_submissions', - 'downloadable', 'id_string', 'is_merged_dataset'), - to_attr='xforms_prefetch') - ).prefetch_related('tags')\ - .prefetch_related(Prefetch( - 'projectuserobjectpermission_set', - queryset=ProjectUserObjectPermission.objects.select_related( - 'user__profile__organizationprofile', - 'permission' + + return ( + super(PrefetchManager, self) + .get_queryset() + .select_related("created_by", "organization") + .prefetch_related( + Prefetch( + "xform_set", + queryset=XForm.objects.filter(deleted_at__isnull=True) + .select_related("user") + .prefetch_related("user") + .prefetch_related("dataview_set") + .prefetch_related("metadata_set") + .only( + "id", + "user", + "project", + "title", + "date_created", + "last_submission_time", + "num_of_submissions", + "downloadable", + "id_string", + "is_merged_dataset", + ), + to_attr="xforms_prefetch", + ) + ) + .prefetch_related("tags") + .prefetch_related( + Prefetch( + "projectuserobjectpermission_set", + queryset=ProjectUserObjectPermission.objects.select_related( + "user__profile__organizationprofile", "permission" + ), + ) + ) + .prefetch_related( + Prefetch( + "projectgroupobjectpermission_set", + queryset=ProjectGroupObjectPermission.objects.select_related( + "group", "permission" + ), ) - ))\ - .prefetch_related(Prefetch( - 'projectgroupobjectpermission_set', - queryset=ProjectGroupObjectPermission.objects.select_related( - 'group', - 'permission' + ) + .prefetch_related("user_stars") + .prefetch_related( + Prefetch( + "organization__team_set", + queryset=Team.objects.all().prefetch_related("user_set"), ) - )).prefetch_related('user_stars')\ - .prefetch_related(Prefetch( - 'organization__team_set', - queryset=Team.objects.all().prefetch_related('user_set') - )) + ) + ) @python_2_unicode_compatible @@ -67,48 +88,56 @@ class Project(BaseModel): name = models.CharField(max_length=255) metadata = JSONField(default=dict) organization = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_org', - on_delete=models.CASCADE) + settings.AUTH_USER_MODEL, related_name="project_org", on_delete=models.CASCADE + ) created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_owner', - on_delete=models.CASCADE) - user_stars = models.ManyToManyField(settings.AUTH_USER_MODEL, - related_name='project_stars') + settings.AUTH_USER_MODEL, related_name="project_owner", on_delete=models.CASCADE + ) + user_stars = models.ManyToManyField( + settings.AUTH_USER_MODEL, related_name="project_stars" + ) shared = models.BooleanField(default=False) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(User, related_name='project_deleted_by', - blank=True, null=True, default=None, - on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, + related_name="project_deleted_by", + blank=True, + null=True, + default=None, + on_delete=models.SET_NULL, + ) objects = models.Manager() - tags = TaggableManager(related_name='project_tags') + tags = TaggableManager(related_name="project_tags") prefetched = PrefetchManager() class Meta: - app_label = 'logger' - unique_together = (('name', 'organization'),) + app_label = "logger" + unique_together = (("name", "organization"),) permissions = ( - ('add_project_xform', "Can add xform to project"), + ("add_project_xform", "Can add xform to project"), ("report_project_xform", "Can make submissions to the project"), - ('transfer_project', "Can transfer project to different owner"), - ('can_export_project_data', "Can export data in project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), ("view_project_all", "Can view all associated data"), ("view_project_data", "Can view submitted data"), ) def __str__(self): - return u'%s|%s' % (self.organization, self.name) + return "%s|%s" % (self.organization, self.name) def clean(self): # pylint: disable=E1101 - query_set = Project.objects.exclude(pk=self.pk)\ - .filter(name__iexact=self.name, organization=self.organization) + query_set = Project.objects.exclude(pk=self.pk).filter( + name__iexact=self.name, organization=self.organization + ) if query_set.exists(): - raise ValidationError(u'Project name "%s" is already in' - u' use in this account.' - % self.name.lower()) + raise ValidationError( + 'Project name "%s" is already in' + " use in this account." % self.name.lower() + ) @property def user(self): @@ -124,7 +153,7 @@ def soft_delete(self, user=None): """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix if user is not None: @@ -140,9 +169,10 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): for perm in get_perms_for_model(Project): assign_perm(perm.codename, instance.organization, instance) - owners = instance.organization.team_set\ - .filter(name="{}#{}".format(instance.organization.username, - OWNER_TEAM_NAME), organization=instance.organization) + owners = instance.organization.team_set.filter( + name="{}#{}".format(instance.organization.username, OWNER_TEAM_NAME), + organization=instance.organization, + ) for owner in owners: assign_perm(perm.codename, owner, instance) @@ -153,8 +183,11 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -post_save.connect(set_object_permissions, sender=Project, - dispatch_uid='set_project_object_permissions') +post_save.connect( + set_object_permissions, + sender=Project, + dispatch_uid="set_project_object_permissions", +) class ProjectUserObjectPermission(UserObjectPermissionBase): diff --git a/onadata/apps/logger/models/survey_type.py b/onadata/apps/logger/models/survey_type.py index 6201c85b7d..d766b35b96 100644 --- a/onadata/apps/logger/models/survey_type.py +++ b/onadata/apps/logger/models/survey_type.py @@ -3,7 +3,7 @@ Survey type model class """ from django.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible @python_2_unicode_compatible @@ -11,10 +11,11 @@ class SurveyType(models.Model): """ Survey type model class """ + slug = models.CharField(max_length=100, unique=True) class Meta: - app_label = 'logger' + app_label = "logger" def __str__(self): return "SurveyType: %s" % self.slug diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index ea8b84296f..6f3503fa64 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -17,57 +17,71 @@ from django.db.models.signals import post_delete, post_save, pre_save from django.urls import reverse from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from future.utils import iteritems -from future.utils import listvalues +from six import iteritems, itervalues, python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase -from past.builtins import cmp -from pyxform import ( - SurveyElementBuilder, constants, create_survey_element_from_dict) +from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question from pyxform.section import RepeatingSection from pyxform.xform2json import create_survey_element_from_xml from taggit.managers import TaggableManager -from onadata.apps.logger.xform_instance_parser import (XLSFormError, - clean_and_parse_xml) +from onadata.apps.logger.xform_instance_parser import XLSFormError, clean_and_parse_xml from django.utils.html import conditional_escape from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.cache_tools import ( - IS_ORG, PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, - PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, XFORM_COUNT, - PROJ_OWNER_CACHE, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) -from onadata.libs.utils.common_tags import (DURATION, ID, KNOWN_MEDIA_TYPES, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, - NOTES, SUBMISSION_TIME, - SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, REVIEW_STATUS, - REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, - DATE_MODIFIED) + IS_ORG, + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + PROJ_OWNER_CACHE, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + DURATION, + ID, + KNOWN_MEDIA_TYPES, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + NOTES, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + REVIEW_STATUS, + REVIEW_COMMENT, + MULTIPLE_SELECT_TYPE, + DATE_MODIFIED, +) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _encode_for_mongo QUESTION_TYPES_TO_EXCLUDE = [ - u'note', + "note", ] XFORM_TITLE_LENGTH = 255 title_pattern = re.compile(r"(.*?)") +cmp = lambda x, y: (x > y) - (x < y) + + def question_types_to_exclude(_type): return _type in QUESTION_TYPES_TO_EXCLUDE def upload_to(instance, filename): - return os.path.join(instance.user.username, 'xls', - os.path.split(filename)[1]) + return os.path.join(instance.user.username, "xls", os.path.split(filename)[1]) -def contains_xml_invalid_char(text, invalids=['&', '>', '<']): +def contains_xml_invalid_char(text, invalids=["&", ">", "<"]): """Check whether 'text' contains ANY invalid xml chars""" return 1 in [c in text for c in invalids] @@ -79,21 +93,22 @@ def set_dict_iterator(self, dict_iterator): # Every section will get its own table # I need to think of an easy way to flatten out a dictionary # parent name, index, table name, data - def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, - parent_index): + def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_index): if table_name not in obs: obs[table_name] = [] this_index = len(obs[table_name]) - obs[table_name].append({ - u"_parent_table_name": parent_table_name, - u"_parent_index": parent_index, - }) + obs[table_name].append( + { + "_parent_table_name": parent_table_name, + "_parent_index": parent_index, + } + ) for (k, v) in iteritems(d): if isinstance(v, dict) and isinstance(v, list): if k in obs[table_name][-1]: raise AssertionError() obs[table_name][-1][k] = v - obs[table_name][-1][u"_index"] = this_index + obs[table_name][-1]["_index"] = this_index for (k, v) in iteritems(d): if isinstance(v, dict): @@ -102,7 +117,7 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, "obs": obs, "table_name": k, "parent_table_name": table_name, - "parent_index": this_index + "parent_index": this_index, } self._build_obs_from_dict(**kwargs) elif isinstance(v, list): @@ -125,7 +140,7 @@ def get_observation_from_dict(self, d): "d": d[root_name], "obs": {}, "table_name": root_name, - "parent_table_name": u"", + "parent_table_name": "", "parent_index": -1, } @@ -142,10 +157,13 @@ def get_forms_shared_with_user(user): """ xforms = XForm.objects.filter( pk__in=user.xformuserobjectpermission_set.values_list( - 'content_object_id', flat=True).distinct(), - downloadable=True, deleted_at__isnull=True) + "content_object_id", flat=True + ).distinct(), + downloadable=True, + deleted_at__isnull=True, + ) - return xforms.exclude(user=user).select_related('user') + return xforms.exclude(user=user).select_related("user") def check_version_set(survey): @@ -158,17 +176,14 @@ def check_version_set(survey): survey_json = json.loads(survey.to_json()) if not survey_json.get("version"): # set utc time as the default version - survey_json['version'] = \ - datetime.utcnow().strftime("%Y%m%d%H%M") + survey_json["version"] = datetime.utcnow().strftime("%Y%m%d%H%M") builder = SurveyElementBuilder() - survey = builder.create_survey_element_from_json( - json.dumps(survey_json)) + survey = builder.create_survey_element_from_json(json.dumps(survey_json)) return survey def _expand_select_all_that_apply(d, key, e): - if e and e.bind.get(u"type") == u"string"\ - and e.type == MULTIPLE_SELECT_TYPE: + if e and e.bind.get("type") == "string" and e.type == MULTIPLE_SELECT_TYPE: options_selected = d[key].split() for child in e.children: new_key = child.get_abbreviated_xpath() @@ -179,9 +194,9 @@ def _expand_select_all_that_apply(d, key, e): class XFormMixin(object): - GEODATA_SUFFIXES = ['latitude', 'longitude', 'altitude', 'precision'] + GEODATA_SUFFIXES = ["latitude", "longitude", "altitude", "precision"] - PREFIX_NAME_REGEX = re.compile(r'(?P.+/)(?P[^/]+)$') + PREFIX_NAME_REGEX = re.compile(r"(?P.+/)(?P[^/]+)$") def _set_uuid_in_xml(self, file_name=None): """ @@ -194,49 +209,55 @@ def _set_uuid_in_xml(self, file_name=None): doc = clean_and_parse_xml(self.xml) model_nodes = doc.getElementsByTagName("model") if len(model_nodes) != 1: - raise Exception(u"xml contains multiple model nodes") + raise Exception("xml contains multiple model nodes") model_node = model_nodes[0] instance_nodes = [ - node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and not node.hasAttribute("id") + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") ] if len(instance_nodes) != 1: - raise Exception(u"Multiple instance nodes without the id " - u"attribute, can't tell which is the main one") + raise Exception( + "Multiple instance nodes without the id " + "attribute, can't tell which is the main one" + ) instance_node = instance_nodes[0] # get the first child whose id attribute matches our id_string survey_nodes = [ - node for node in instance_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - (node.tagName == file_name or node.attributes.get('id')) + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and (node.tagName == file_name or node.attributes.get("id")) ] if len(survey_nodes) != 1: - raise Exception( - u"Multiple survey nodes with the id '%s'" % self.id_string) + raise Exception("Multiple survey nodes with the id '%s'" % self.id_string) survey_node = survey_nodes[0] formhub_nodes = [ - n for n in survey_node.childNodes + n + for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "formhub" ] if len(formhub_nodes) > 1: - raise Exception( - u"Multiple formhub nodes within main instance node") + raise Exception("Multiple formhub nodes within main instance node") elif len(formhub_nodes) == 1: formhub_node = formhub_nodes[0] else: formhub_node = survey_node.insertBefore( - doc.createElement("formhub"), survey_node.firstChild) + doc.createElement("formhub"), survey_node.firstChild + ) uuid_nodes = [ - node for node in formhub_node.childNodes + node + for node in formhub_node.childNodes if node.nodeType == Node.ELEMENT_NODE and node.tagName == "uuid" ] @@ -246,22 +267,25 @@ def _set_uuid_in_xml(self, file_name=None): # append the calculate bind node calculate_node = doc.createElement("bind") calculate_node.setAttribute( - "nodeset", "/%s/formhub/uuid" % survey_node.tagName) + "nodeset", "/%s/formhub/uuid" % survey_node.tagName + ) calculate_node.setAttribute("type", "string") calculate_node.setAttribute("calculate", "'%s'" % self.uuid) model_node.appendChild(calculate_node) - self.xml = doc.toprettyxml(indent=" ", encoding='utf-8') + self.xml = doc.toprettyxml(indent=" ", encoding="utf-8") # hack # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ - text_re = re.compile('(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*') - pretty_xml = text_re.sub(lambda m: ''.join(m.group(1, 2, 3)), - self.xml.decode('utf-8')) - inline_output = output_re.sub('\g<1>', pretty_xml) # noqa - inline_output = re.compile('').sub( # noqa - '', inline_output) + text_re = re.compile("(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*") + pretty_xml = text_re.sub( + lambda m: "".join(m.group(1, 2, 3)), self.xml.decode("utf-8") + ) + inline_output = output_re.sub("\g<1>", pretty_xml) # noqa + inline_output = re.compile("").sub( # noqa + "", inline_output + ) self.xml = inline_output class Meta: @@ -270,7 +294,7 @@ class Meta: @property def has_id_string_changed(self): - return getattr(self, '_id_string_changed', False) + return getattr(self, "_id_string_changed", False) def add_instances(self): _get_observation_from_dict = DictOrganizer().get_observation_from_dict @@ -293,8 +317,8 @@ def get_unique_id_string(self, id_string, count=0): # id_string already existed if self._id_string_already_exists_in_account(id_string): if count != 0: - if re.match(r'\w+_\d+$', id_string): - a = id_string.split('_') + if re.match(r"\w+_\d+$", id_string): + a = id_string.split("_") id_string = "_".join(a[:-1]) count += 1 id_string = "{}_{}".format(id_string, count) @@ -307,10 +331,9 @@ def get_survey(self): if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() - self._survey = \ - builder.create_survey_element_from_json(self.json) + self._survey = builder.create_survey_element_from_json(self.json) except ValueError: - xml = b(bytearray(self.xml, encoding='utf-8')) + xml = b(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) return self._survey @@ -331,8 +354,7 @@ def get_survey_element(self, name_or_xpath): # search by name if xpath fails fields = [ - field for field in self.get_survey_elements() - if field.name == name_or_xpath + field for field in self.get_survey_elements() if field.name == name_or_xpath ] return fields[0] if len(fields) else None @@ -343,16 +365,17 @@ def get_child_elements(self, name_or_xpath, split_select_multiples=True): appended to the list. If the name_or_xpath is a repeat we iterate through the child elements as well. """ - GROUP_AND_SELECT_MULTIPLES = ['group'] + GROUP_AND_SELECT_MULTIPLES = ["group"] if split_select_multiples: - GROUP_AND_SELECT_MULTIPLES += ['select all that apply'] + GROUP_AND_SELECT_MULTIPLES += ["select all that apply"] def flatten(elem, items=[]): results = [] if elem: xpath = elem.get_abbreviated_xpath() - if elem.type in GROUP_AND_SELECT_MULTIPLES or \ - (xpath == name_or_xpath and elem.type == 'repeat'): + if elem.type in GROUP_AND_SELECT_MULTIPLES or ( + xpath == name_or_xpath and elem.type == "repeat" + ): for child in elem.children: results += flatten(child) else: @@ -364,16 +387,14 @@ def flatten(elem, items=[]): return flatten(element) - def get_choice_label(self, field, choice_value, lang='English'): - choices = [ - choice for choice in field.children if choice.name == choice_value - ] + def get_choice_label(self, field, choice_value, lang="English"): + choices = [choice for choice in field.children if choice.name == choice_value] if len(choices): choice = choices[0] label = choice.label if isinstance(label, dict): - label = label.get(lang, listvalues(choice.label)[0]) + label = label.get(lang, itervalues(choice.label)[0]) return label @@ -386,24 +407,27 @@ def get_mongo_field_names_dict(self): """ names = {} for elem in self.get_survey_elements(): - names[_encode_for_mongo(text(elem.get_abbreviated_xpath()))] = \ - elem.get_abbreviated_xpath() + names[ + _encode_for_mongo(text(elem.get_abbreviated_xpath())) + ] = elem.get_abbreviated_xpath() return names survey_elements = property(get_survey_elements) def get_field_name_xpaths_only(self): return [ - elem.get_abbreviated_xpath() for elem in self.survey_elements - if elem.type != '' and elem.type != 'survey' + elem.get_abbreviated_xpath() + for elem in self.survey_elements + if elem.type != "" and elem.type != "survey" ] def geopoint_xpaths(self): survey_elements = self.get_survey_elements() return [ - e.get_abbreviated_xpath() for e in survey_elements - if e.bind.get(u'type') == u'geopoint' + e.get_abbreviated_xpath() + for e in survey_elements + if e.bind.get("type") == "geopoint" ] def xpath_of_first_geopoint(self): @@ -411,11 +435,7 @@ def xpath_of_first_geopoint(self): return len(geo_xpaths) and geo_xpaths[0] - def xpaths(self, - prefix='', - survey_element=None, - result=None, - repeat_iterations=4): + def xpaths(self, prefix="", survey_element=None, result=None, repeat_iterations=4): """ Return a list of XPaths for this survey that will be used as headers for the csv export. @@ -426,13 +446,15 @@ def xpaths(self, return [] result = [] if result is None else result - path = '/'.join([prefix, text(survey_element.name)]) + path = "/".join([prefix, text(survey_element.name)]) if survey_element.children is not None: # add xpaths to result for each child - indices = [''] if not isinstance(survey_element, - RepeatingSection) else \ - ['[%d]' % (i + 1) for i in range(repeat_iterations)] + indices = ( + [""] + if not isinstance(survey_element, RepeatingSection) + else ["[%d]" % (i + 1) for i in range(repeat_iterations)] + ) for i in indices: for e in survey_element.children: self.xpaths(path + i, e, result, repeat_iterations) @@ -442,12 +464,14 @@ def xpaths(self, # replace the single question column with a column for each # item in a select all that apply question. - if survey_element.bind.get(u'type') == u'string' \ - and survey_element.type == MULTIPLE_SELECT_TYPE: + if ( + survey_element.bind.get("type") == "string" + and survey_element.type == MULTIPLE_SELECT_TYPE + ): result.pop() for child in survey_element.children: - result.append('/'.join([path, child.name])) - elif survey_element.bind.get(u'type') == u'geopoint': + result.append("/".join([path, child.name])) + elif survey_element.bind.get("type") == "geopoint": result += self.get_additional_geopoint_xpaths(path) return result @@ -461,39 +485,50 @@ def get_additional_geopoint_xpaths(cls, xpath): DataDictionary.GEODATA_SUFFIXES """ match = cls.PREFIX_NAME_REGEX.match(xpath) - prefix = '' + prefix = "" name = xpath if match: - prefix = match.groupdict()['prefix'] - name = match.groupdict()['name'] + prefix = match.groupdict()["prefix"] + name = match.groupdict()["name"] - return [ - '_'.join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES - ] + return ["_".join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES] def _additional_headers(self): return [ - u'_xform_id_string', u'_percentage_complete', u'_status', - u'_attachments', u'_potential_duplicates' + "_xform_id_string", + "_percentage_complete", + "_status", + "_attachments", + "_potential_duplicates", ] - def get_headers( - self, include_additional_headers=False, repeat_iterations=4): + def get_headers(self, include_additional_headers=False, repeat_iterations=4): """ Return a list of headers for a csv file. """ + def shorten(xpath): - xpath_list = xpath.split('/') - return '/'.join(xpath_list[2:]) + xpath_list = xpath.split("/") + return "/".join(xpath_list[2:]) header_list = [ - shorten(xpath) for xpath in self.xpaths( - repeat_iterations=repeat_iterations)] + shorten(xpath) for xpath in self.xpaths(repeat_iterations=repeat_iterations) + ] header_list += [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, - REVIEW_STATUS, REVIEW_COMMENT, VERSION, DURATION, - SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + REVIEW_STATUS, + REVIEW_COMMENT, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, ] if include_additional_headers: header_list += self._additional_headers() @@ -501,7 +536,7 @@ def shorten(xpath): def get_keys(self): def remove_first_index(xpath): - return re.sub(r'\[1\]', '', xpath) + return re.sub(r"\[1\]", "", xpath) return [remove_first_index(header) for header in self.get_headers()] @@ -512,15 +547,14 @@ def get_element(self, abbreviated_xpath): self._survey_elements[e.get_abbreviated_xpath()] = e def remove_all_indices(xpath): - return re.sub(r"\[\d+\]", u"", xpath) + return re.sub(r"\[\d+\]", "", xpath) clean_xpath = remove_all_indices(abbreviated_xpath) return self._survey_elements.get(clean_xpath) def get_default_language(self): - if not hasattr(self, '_default_language'): - self._default_language = \ - self.survey.to_json_dict().get('default_language') + if not hasattr(self, "_default_language"): + self._default_language = self.survey.to_json_dict().get("default_language") return self._default_language @@ -547,21 +581,19 @@ def get_label(self, abbreviated_xpath, elem=None, language=None): label = label[language] else: language = self.get_language(list(label)) - label = label[language] if language else '' + label = label[language] if language else "" return label def get_xpath_cmp(self): if not hasattr(self, "_xpaths"): - self._xpaths = [ - e.get_abbreviated_xpath() for e in self.survey_elements - ] + self._xpaths = [e.get_abbreviated_xpath() for e in self.survey_elements] def xpath_cmp(x, y): # For the moment, we aren't going to worry about repeating # nodes. - new_x = re.sub(r"\[\d+\]", u"", x) - new_y = re.sub(r"\[\d+\]", u"", y) + new_x = re.sub(r"\[\d+\]", "", x) + new_y = re.sub(r"\[\d+\]", "", y) if new_x == new_y: return cmp(x, y) if new_x not in self._xpaths and new_y not in self._xpaths: @@ -612,7 +644,7 @@ def _rename_key(self, d, old_key, new_key): del d[old_key] def _expand_geocodes(self, d, key, e): - if e and e.bind.get(u"type") == u"geopoint": + if e and e.bind.get("type") == "geopoint": geodata = d[key].split() for i in range(len(geodata)): new_key = "%s_%s" % (key, self.geodata_suffixes[i]) @@ -634,15 +666,11 @@ def _mark_start_time_boolean(self): self.has_start_time = False def get_survey_elements_of_type(self, element_type): - return [ - e for e in self.get_survey_elements() if e.type == element_type - ] + return [e for e in self.get_survey_elements() if e.type == element_type] def get_survey_elements_with_choices(self): - if not hasattr(self, '_survey_elements_with_choices'): - choices_type = [ - constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY - ] + if not hasattr(self, "_survey_elements_with_choices"): + choices_type = [constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY] self._survey_elements_with_choices = [ e for e in self.get_survey_elements() if e.type in choices_type @@ -654,11 +682,17 @@ def get_select_one_xpaths(self): """ Returns abbreviated_xpath for SELECT_ONE questions in the survey. """ - if not hasattr(self, '_select_one_xpaths'): + if not hasattr(self, "_select_one_xpaths"): self._select_one_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ONE]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ONE] + ], + [], + ) + ] return self._select_one_xpaths @@ -667,20 +701,26 @@ def get_select_multiple_xpaths(self): Returns abbreviated_xpath for SELECT_ALL_THAT_APPLY questions in the survey. """ - if not hasattr(self, '_select_multiple_xpaths'): + if not hasattr(self, "_select_multiple_xpaths"): self._select_multiple_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ALL_THAT_APPLY]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ALL_THAT_APPLY] + ], + [], + ) + ] return self._select_multiple_xpaths def get_media_survey_xpaths(self): return [ e.get_abbreviated_xpath() - for e in sum([ - self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES - ], []) + for e in sum( + [self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES], [] + ) ] def get_osm_survey_xpaths(self): @@ -689,93 +729,102 @@ def get_osm_survey_xpaths(self): """ return [ elem.get_abbreviated_xpath() - for elem in self.get_survey_elements_of_type('osm')] + for elem in self.get_survey_elements_of_type("osm") + ] @python_2_unicode_compatible class XForm(XFormMixin, BaseModel): - CLONED_SUFFIX = '_cloned' + CLONED_SUFFIX = "_cloned" MAX_ID_LENGTH = 100 xls = models.FileField(upload_to=upload_to, null=True) - json = models.TextField(default=u'') - description = models.TextField(default=u'', null=True, blank=True) + json = models.TextField(default="") + description = models.TextField(default="", null=True, blank=True) xml = models.TextField() user = models.ForeignKey( - User, related_name='xforms', null=True, on_delete=models.CASCADE) + User, related_name="xforms", null=True, on_delete=models.CASCADE + ) require_auth = models.BooleanField(default=False) shared = models.BooleanField(default=False) shared_data = models.BooleanField(default=False) downloadable = models.BooleanField(default=True) allows_sms = models.BooleanField(default=False) encrypted = models.BooleanField(default=False) - deleted_by = models.ForeignKey(User, related_name='xform_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + User, + related_name="xform_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) # the following fields are filled in automatically sms_id_string = models.SlugField( editable=False, verbose_name=ugettext_lazy("SMS ID"), max_length=MAX_ID_LENGTH, - default='') + default="", + ) id_string = models.SlugField( - editable=False, - verbose_name=ugettext_lazy("ID"), - max_length=MAX_ID_LENGTH) + editable=False, verbose_name=ugettext_lazy("ID"), max_length=MAX_ID_LENGTH + ) title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) last_submission_time = models.DateTimeField(blank=True, null=True) has_start_time = models.BooleanField(default=False) - uuid = models.CharField(max_length=36, default=u'', db_index=True) - public_key = models.TextField(default='', blank=True, null=True) + uuid = models.CharField(max_length=36, default="", db_index=True) + public_key = models.TextField(default="", blank=True, null=True) - uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', - re.DOTALL) - instance_id_regex = re.compile(r'.*?id="([^"]+)".*', - re.DOTALL) + uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', re.DOTALL) + instance_id_regex = re.compile(r'.*?id="([^"]+)".*', re.DOTALL) uuid_node_location = 2 uuid_bind_location = 4 - bamboo_dataset = models.CharField(max_length=60, default=u'') + bamboo_dataset = models.CharField(max_length=60, default="") instances_with_geopoints = models.BooleanField(default=False) instances_with_osm = models.BooleanField(default=False) num_of_submissions = models.IntegerField(default=0) - version = models.CharField( - max_length=XFORM_TITLE_LENGTH, null=True, blank=True) - project = models.ForeignKey('Project', on_delete=models.CASCADE) + version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True, blank=True) + project = models.ForeignKey("Project", on_delete=models.CASCADE) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) metadata_set = GenericRelation( - 'main.MetaData', - content_type_field='content_type_id', - object_id_field="object_id") + "main.MetaData", + content_type_field="content_type_id", + object_id_field="object_id", + ) has_hxl_support = models.BooleanField(default=False) last_updated_at = models.DateTimeField(auto_now=True) - hash = models.CharField(_("Hash"), max_length=36, blank=True, null=True, - default=None) + hash = models.CharField( + _("Hash"), max_length=36, blank=True, null=True, default=None + ) # XForm was created as a merged dataset is_merged_dataset = models.BooleanField(default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = (("user", "id_string", "project"), - ("user", "sms_id_string", "project")) + app_label = "logger" + unique_together = ( + ("user", "id_string", "project"), + ("user", "sms_id_string", "project"), + ) verbose_name = ugettext_lazy("XForm") verbose_name_plural = ugettext_lazy("XForms") - ordering = ("pk", ) + ordering = ("pk",) permissions = ( ("view_xform_all", _("Can view all associated data")), ("view_xform_data", _("Can view submitted data")), ("report_xform", _("Can make submissions to the form")), - ("move_xform", _(u"Can move form between projects")), - ("transfer_xform", _(u"Can transfer form ownership.")), - ("can_export_xform_data", _(u"Can export form data")), - ("delete_submission", _(u"Can delete submissions from form")), + ("move_xform", _("Can move form between projects")), + ("transfer_xform", _("Can transfer form ownership.")), + ("can_export_xform_data", _("Can export form data")), + ("delete_submission", _("Can delete submissions from form")), ) def file_name(self): @@ -784,10 +833,8 @@ def file_name(self): def url(self): return reverse( "download_xform", - kwargs={ - "username": self.user.username, - "id_string": self.id_string - }) + kwargs={"username": self.user.username, "id_string": self.id_string}, + ) @property def has_instances_with_geopoints(self): @@ -809,24 +856,25 @@ def _set_title(self): if matches: title_xml = matches[0][:XFORM_TITLE_LENGTH] else: - title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else '' + title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else "" if self.title and title_xml != self.title: title_xml = self.title[:XFORM_TITLE_LENGTH] if isinstance(self.xml, b): - self.xml = self.xml.decode('utf-8') - self.xml = title_pattern.sub(u"%s" % title_xml, - self.xml) + self.xml = self.xml.decode("utf-8") + self.xml = title_pattern.sub("%s" % title_xml, self.xml) self._set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( - _("Title shouldn't have any invalid xml " - "characters ('>' '&' '<')")) + _("Title shouldn't have any invalid xml " "characters ('>' '&' '<')") + ) # Capture urls within form title - if re.search(r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", self.title): # noqa - raise XLSFormError( - _("Invalid title value; value shouldn't match a URL")) + if re.search( + r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", + self.title, + ): # noqa + raise XLSFormError(_("Invalid title value; value shouldn't match a URL")) self.title = title_xml @@ -834,15 +882,15 @@ def _set_hash(self): self.hash = self.get_hash() def _set_encrypted_field(self): - if self.json and self.json != '': + if self.json and self.json != "": json_dict = json.loads(self.json) - self.encrypted = 'public_key' in json_dict + self.encrypted = "public_key" in json_dict def _set_public_key_field(self): - if self.json and self.json != '': + if self.json and self.json != "": if self.num_of_submissions == 0 and self.public_key: json_dict = json.loads(self.json) - json_dict['public_key'] = self.public_key + json_dict["public_key"] = self.public_key survey = create_survey_element_from_dict(json_dict) self.json = survey.to_json() self.xml = survey.to_xml() @@ -852,71 +900,84 @@ def update(self, *args, **kwargs): super(XForm, self).save(*args, **kwargs) def save(self, *args, **kwargs): - update_fields = kwargs.get('update_fields') + update_fields = kwargs.get("update_fields") if update_fields: - kwargs['update_fields'] = list( - set(list(update_fields) + ['date_modified'])) - if update_fields is None or 'title' in update_fields: + kwargs["update_fields"] = list(set(list(update_fields) + ["date_modified"])) + if update_fields is None or "title" in update_fields: self._set_title() if self.pk is None: self._set_hash() - if update_fields is None or 'encrypted' in update_fields: + if update_fields is None or "encrypted" in update_fields: self._set_encrypted_field() - if update_fields is None or 'id_string' in update_fields: + if update_fields is None or "id_string" in update_fields: old_id_string = self.id_string if not self.deleted_at: self._set_id_string() # check if we have an existing id_string, # if so, the one must match but only if xform is NOT new - if self.pk and old_id_string and old_id_string != self.id_string \ - and self.num_of_submissions > 0: + if ( + self.pk + and old_id_string + and old_id_string != self.id_string + and self.num_of_submissions > 0 + ): raise XLSFormError( - _(u"Your updated form's id_string '%(new_id)s' must match " - "the existing forms' id_string '%(old_id)s'." % - {'new_id': self.id_string, - 'old_id': old_id_string})) - - if getattr(settings, 'STRICT', True) and \ - not re.search(r"^[\w-]+$", self.id_string): + _( + "Your updated form's id_string '%(new_id)s' must match " + "the existing forms' id_string '%(old_id)s'." + % {"new_id": self.id_string, "old_id": old_id_string} + ) + ) + + if getattr(settings, "STRICT", True) and not re.search( + r"^[\w-]+$", self.id_string + ): raise XLSFormError( - _(u'In strict mode, the XForm ID must be a ' - 'valid slug and contain no spaces. Please ensure' - ' that you have set an id_string in the settings sheet ' - 'or have modified the filename to not contain' - ' any spaces.')) - - if not self.sms_id_string and (update_fields is None or - 'id_string' in update_fields): + _( + "In strict mode, the XForm ID must be a " + "valid slug and contain no spaces. Please ensure" + " that you have set an id_string in the settings sheet " + "or have modified the filename to not contain" + " any spaces." + ) + ) + + if not self.sms_id_string and ( + update_fields is None or "id_string" in update_fields + ): try: # try to guess the form's wanted sms_id_string # from it's json rep (from XLSForm) # otherwise, use id_string to ensure uniqueness self.sms_id_string = json.loads(self.json).get( - 'sms_keyword', self.id_string) + "sms_keyword", self.id_string + ) except Exception: self.sms_id_string = self.id_string - if update_fields is None or 'public_key' in update_fields: + if update_fields is None or "public_key" in update_fields: self._set_public_key_field() - if 'skip_xls_read' in kwargs: - del kwargs['skip_xls_read'] + if "skip_xls_read" in kwargs: + del kwargs["skip_xls_read"] - if (self.id_string and len( - self.id_string) > self.MAX_ID_LENGTH) or \ - (self.sms_id_string and len( - self.sms_id_string) > self.MAX_ID_LENGTH): + if (self.id_string and len(self.id_string) > self.MAX_ID_LENGTH) or ( + self.sms_id_string and len(self.sms_id_string) > self.MAX_ID_LENGTH + ): raise XLSFormError( - _(u'The XForm id_string provided exceeds %s characters.' - ' Please change the "id_string" or "form_id" values' - 'in settings sheet or reduce the file name if you do' - ' not have a settings sheets.' % self.MAX_ID_LENGTH)) + _( + "The XForm id_string provided exceeds %s characters." + ' Please change the "id_string" or "form_id" values' + "in settings sheet or reduce the file name if you do" + " not have a settings sheets." % self.MAX_ID_LENGTH + ) + ) is_version_available = self.version is not None if is_version_available and contains_xml_invalid_char(self.version): raise XLSFormError( - _("Version shouldn't have any invalid " - "characters ('>' '&' '<')")) + _("Version shouldn't have any invalid " "characters ('>' '&' '<')") + ) self.description = conditional_escape(self.description) @@ -936,49 +997,56 @@ def soft_delete(self, user=None): """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.id_string += deletion_suffix self.sms_id_string += deletion_suffix self.downloadable = False # only take the first 100 characters (within the set max_length) - self.id_string = self.id_string[:self.MAX_ID_LENGTH] - self.sms_id_string = self.sms_id_string[:self.MAX_ID_LENGTH] - - update_fields = ['date_modified', 'deleted_at', 'id_string', - 'sms_id_string', 'downloadable'] + self.id_string = self.id_string[: self.MAX_ID_LENGTH] + self.sms_id_string = self.sms_id_string[: self.MAX_ID_LENGTH] + + update_fields = [ + "date_modified", + "deleted_at", + "id_string", + "sms_id_string", + "downloadable", + ] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) # Delete associated filtered datasets for dataview in self.dataview_set.all(): dataview.soft_delete(user) # Delete associated Merged-Datasets - for merged_dataset in self.mergedxform_ptr.filter( - deleted_at__isnull=True): + for merged_dataset in self.mergedxform_ptr.filter(deleted_at__isnull=True): merged_dataset.soft_delete(user) # Delete associated Form Media Files - for metadata in self.metadata_set.filter( - deleted_at__isnull=True): + for metadata in self.metadata_set.filter(deleted_at__isnull=True): metadata.soft_delete() def submission_count(self, force_update=False): if self.num_of_submissions == 0 or force_update: if self.is_merged_dataset: - count = self.mergedxform.xforms.aggregate( - num=Sum('num_of_submissions')).get('num') or 0 + count = ( + self.mergedxform.xforms.aggregate( + num=Sum("num_of_submissions") + ).get("num") + or 0 + ) else: count = self.instances.filter(deleted_at__isnull=True).count() if count != self.num_of_submissions: self.num_of_submissions = count - self.save(update_fields=['num_of_submissions']) + self.save(update_fields=["num_of_submissions"]) # clear cache - key = '{}{}'.format(XFORM_COUNT, self.pk) + key = "{}{}".format(XFORM_COUNT, self.pk) safe_delete(key) return self.num_of_submissions @@ -991,23 +1059,28 @@ def submission_count_for_today(self): current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - count = cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") if cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}" - ) == current_date else 0 + datetime(today.year, today.month, today.day) + ).isoformat() + count = ( + cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") + if cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}") + == current_date + else 0 + ) return count def geocoded_submission_count(self): """Number of geocoded submissions.""" return self.instances.filter( - deleted_at__isnull=True, geom__isnull=False).count() + deleted_at__isnull=True, geom__isnull=False + ).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: try: - last_submission = self.instances.\ - filter(deleted_at__isnull=True).latest("date_created") + last_submission = self.instances.filter(deleted_at__isnull=True).latest( + "date_created" + ) except ObjectDoesNotExist: pass else: @@ -1023,7 +1096,7 @@ def time_of_last_submission_update(self): pass def get_hash(self): - return u'md5:%s' % md5(self.xml.encode('utf8')).hexdigest() + return "md5:%s" % md5(self.xml.encode("utf8")).hexdigest() @property def can_be_replaced(self): @@ -1037,8 +1110,7 @@ def public_forms(cls): def update_profile_num_submissions(sender, instance, **kwargs): profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=instance.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=instance.user.profile.pk) except ObjectDoesNotExist: pass else: @@ -1051,15 +1123,16 @@ def update_profile_num_submissions(sender, instance, **kwargs): post_delete.connect( update_profile_num_submissions, sender=XForm, - dispatch_uid='update_profile_num_submissions') + dispatch_uid="update_profile_num_submissions", +) def clear_project_cache(project_id): - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_FORMS_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, project_id)) def set_object_permissions(sender, instance=None, created=False, **kwargs): @@ -1067,30 +1140,31 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): project = instance.project project.refresh_from_db() clear_project_cache(project.pk) - safe_delete('{}{}'.format(IS_ORG, instance.pk)) + safe_delete("{}{}".format(IS_ORG, instance.pk)) if created: from onadata.libs.permissions import OwnerRole + OwnerRole.add(instance.user, instance) if instance.created_by and instance.user != instance.created_by: OwnerRole.add(instance.created_by, instance) from onadata.libs.utils.project_utils import set_project_perms_to_xform + set_project_perms_to_xform(instance, project) post_save.connect( - set_object_permissions, - sender=XForm, - dispatch_uid='xform_object_permissions') + set_object_permissions, sender=XForm, dispatch_uid="xform_object_permissions" +) def save_project(sender, instance=None, created=False, **kwargs): - instance.project.save(update_fields=['date_modified']) + instance.project.save(update_fields=["date_modified"]) -pre_save.connect(save_project, sender=XForm, dispatch_uid='save_project_xform') +pre_save.connect(save_project, sender=XForm, dispatch_uid="save_project_xform") def xform_post_delete_callback(sender, instance, **kwargs): @@ -1099,9 +1173,8 @@ def xform_post_delete_callback(sender, instance, **kwargs): post_delete.connect( - xform_post_delete_callback, - sender=XForm, - dispatch_uid='xform_post_delete_callback') + xform_post_delete_callback, sender=XForm, dispatch_uid="xform_post_delete_callback" +) class XFormUserObjectPermission(UserObjectPermissionBase): @@ -1121,12 +1194,10 @@ def check_xform_uuid(new_uuid): Checks if a new_uuid has already been used, if it has it raises the exception DuplicateUUIDError. """ - count = XForm.objects.filter(uuid=new_uuid, - deleted_at__isnull=True).count() + count = XForm.objects.filter(uuid=new_uuid, deleted_at__isnull=True).count() if count > 0: - raise DuplicateUUIDError( - "An xform with uuid: %s already exists" % new_uuid) + raise DuplicateUUIDError("An xform with uuid: %s already exists" % new_uuid) def update_xform_uuid(username, id_string, new_uuid): diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 58f7cfcc57..8cb0596f70 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -10,7 +10,7 @@ from django.test import RequestFactory from django.urls import reverse from django_digest.test import Client as DigestClient -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from httmock import HTTMock, urlmatch from onadata.apps.logger.models import Instance, XForm @@ -23,28 +23,29 @@ storage = get_storage_class()() -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def form_list_xml(url, request, **kwargs): response = requests.Response() factory = RequestFactory() req = factory.get(url.path) - req.user = authenticate(username='bob', password='bob') + req.user = authenticate(username="bob", password="bob") req.user.profile.require_auth = False req.user.profile.save() - id_string = 'transportation_2011_07_25' - if url.path.endswith('formList'): - res = formList(req, username='bob') - elif url.path.endswith('form.xml'): - res = download_xform(req, username='bob', id_string=id_string) - elif url.path.find('xformsManifest') > -1: - res = xformsManifest(req, username='bob', id_string=id_string) - elif url.path.find('formid-media') > -1: - data_id = url.path[url.path.rfind('/') + 1:] + id_string = "transportation_2011_07_25" + if url.path.endswith("formList"): + res = formList(req, username="bob") + elif url.path.endswith("form.xml"): + res = download_xform(req, username="bob", id_string=id_string) + elif url.path.find("xformsManifest") > -1: + res = xformsManifest(req, username="bob", id_string=id_string) + elif url.path.find("formid-media") > -1: + data_id = url.path[url.path.rfind("/") + 1 :] res = download_media_data( - req, username='bob', id_string=id_string, data_id=data_id) + req, username="bob", id_string=id_string, data_id=data_id + ) response._content = get_streaming_content(res) else: - res = formList(req, username='bob') + res = formList(req, username="bob") response.status_code = 200 if not response._content: response._content = res.content @@ -60,15 +61,15 @@ def get_streaming_content(res): return content -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def instances_xml(url, request, **kwargs): response = requests.Response() client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') - res = client.get('%s?%s' % (url.path, url.query)) + client.set_authorization("bob", "bob", "Digest") + res = client.get("%s?%s" % (url.path, url.query)) if res.status_code == 302: - res = client.get(res['Location']) - response.encoding = res.get('content-type') + res = client.get(res["Location"]) + response.encoding = res.get("content-type") response._content = get_streaming_content(res) else: response._content = res.content @@ -77,27 +78,24 @@ def instances_xml(url, request, **kwargs): class TestBriefcaseClient(TestBase): - def setUp(self): TestBase.setUp(self) self._publish_transportation_form() self._submit_transport_instance_w_attachment() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + uf = UploadedFile(file=open(src, "rb"), content_type="image/png") count = MetaData.objects.count() MetaData.media_upload(self.xform, uf) self.assertEqual(MetaData.objects.count(), count + 1) url = urljoin( - self.base_url, - reverse(profile, kwargs={'username': self.user.username}) + self.base_url, reverse(profile, kwargs={"username": self.user.username}) ) self._logout() - self._create_user_and_login('deno', 'deno') + self._create_user_and_login("deno", "deno") self.bc = BriefcaseClient( - username='bob', password='bob', - url=url, - user=self.user + username="bob", password="bob", url=url, user=self.user ) def test_download_xform_xml(self): @@ -107,14 +105,14 @@ def test_download_xform_xml(self): with HTTMock(form_list_xml): self.bc.download_xforms() forms_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string) + "deno", "briefcase", "forms", self.xform.id_string + ) self.assertTrue(storage.exists(forms_folder_path)) - forms_path = os.path.join(forms_folder_path, - '%s.xml' % self.xform.id_string) + forms_path = os.path.join(forms_folder_path, "%s.xml" % self.xform.id_string) self.assertTrue(storage.exists(forms_path)) - form_media_path = os.path.join(forms_folder_path, 'form-media') + form_media_path = os.path.join(forms_folder_path, "form-media") self.assertTrue(storage.exists(form_media_path)) - media_path = os.path.join(form_media_path, 'screenshot.png') + media_path = os.path.join(form_media_path, "screenshot.png") self.assertTrue(storage.exists(media_path)) """ @@ -123,15 +121,18 @@ def test_download_xform_xml(self): with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) instance_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string, 'instances') + "deno", "briefcase", "forms", self.xform.id_string, "instances" + ) self.assertTrue(storage.exists(instance_folder_path)) instance = Instance.objects.all()[0] instance_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, 'submission.xml') + instance_folder_path, "uuid%s" % instance.uuid, "submission.xml" + ) self.assertTrue(storage.exists(instance_path)) media_file = "1335783522563.jpg" media_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, media_file) + instance_folder_path, "uuid%s" % instance.uuid, media_file + ) self.assertTrue(storage.exists(media_path)) def test_push(self): @@ -140,22 +141,22 @@ def test_push(self): with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) XForm.objects.all().delete() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 0) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 0) self.bc.push() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 1) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 1) def tearDown(self): # remove media files - for username in ['bob', 'deno']: + for username in ["bob", "deno"]: if storage.exists(username): shutil.rmtree(storage.path(username)) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index 652ad42436..d6ceace3ce 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -2,7 +2,7 @@ import re import dateutil.parser from builtins import str as text -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from xml.dom import minidom, Node from django.utils.encoding import smart_text, smart_str @@ -18,25 +18,25 @@ class XLSFormError(Exception): @python_2_unicode_compatible class DuplicateInstance(Exception): def __str__(self): - return _(u'Duplicate Instance') + return _("Duplicate Instance") @python_2_unicode_compatible class InstanceInvalidUserError(Exception): def __str__(self): - return _(u'Could not determine the user.') + return _("Could not determine the user.") @python_2_unicode_compatible class InstanceParseError(Exception): def __str__(self): - return _(u'The instance could not be parsed.') + return _("The instance could not be parsed.") @python_2_unicode_compatible class InstanceEmptyError(InstanceParseError): def __str__(self): - return _(u'Empty instance') + return _("Empty instance") class InstanceFormatError(Exception): @@ -67,25 +67,31 @@ def get_meta_from_xml(xml_str, meta_name): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - meta_tags = [n for n in survey_node.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == "meta" or - n.tagName.lower() == "orx:meta")] + meta_tags = [ + n + for n in survey_node.childNodes + if n.nodeType == Node.ELEMENT_NODE + and (n.tagName.lower() == "meta" or n.tagName.lower() == "orx:meta") + ] if len(meta_tags) == 0: return None # get the requested tag meta_tag = meta_tags[0] - uuid_tags = [n for n in meta_tag.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == meta_name.lower() or - n.tagName.lower() == u'orx:%s' % meta_name.lower())] + uuid_tags = [ + n + for n in meta_tag.childNodes + if n.nodeType == Node.ELEMENT_NODE + and ( + n.tagName.lower() == meta_name.lower() + or n.tagName.lower() == "orx:%s" % meta_name.lower() + ) + ] if len(uuid_tags) == 0: return None uuid_tag = uuid_tags[0] - return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild\ - else None + return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild else None def get_uuid_from_xml(xml): @@ -94,6 +100,7 @@ def _uuid_only(uuid, regex): if matches and len(matches.groups()) > 0: return matches.groups()[0] return None + uuid = get_meta_from_xml(xml, "instanceID") regex = re.compile(r"uuid:(.*)") if uuid: @@ -106,8 +113,8 @@ def _uuid_only(uuid, regex): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - uuid = survey_node.getAttribute('instanceID') - if uuid != '': + uuid = survey_node.getAttribute("instanceID") + if uuid != "": return _uuid_only(uuid, regex) return None @@ -121,8 +128,8 @@ def get_submission_date_from_xml(xml): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - submissionDate = survey_node.getAttribute('submissionDate') - if submissionDate != '': + submissionDate = survey_node.getAttribute("submissionDate") + if submissionDate != "": return dateutil.parser.parse(submissionDate) return None @@ -139,7 +146,7 @@ def get_deprecated_uuid_from_xml(xml): def clean_and_parse_xml(xml_string): clean_xml_str = xml_string.strip() - clean_xml_str = re.sub(r">\s+<", u"><", smart_text(clean_xml_str)) + clean_xml_str = re.sub(r">\s+<", "><", smart_text(clean_xml_str)) xml_obj = minidom.parseString(smart_str(clean_xml_str)) return xml_obj @@ -148,8 +155,7 @@ def _xml_node_to_dict(node, repeats=[], encrypted=False): if len(node.childNodes) == 0: # there's no data for this leaf node return None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node return {node.nodeName: node.childNodes[0].nodeValue} else: @@ -173,7 +179,7 @@ def _xml_node_to_dict(node, repeats=[], encrypted=False): node_type = dict # check if name is in list of repeats and make it a list if so # All the photo attachments in an encrypted form use name media - if child_xpath in repeats or (encrypted and child_name == 'media'): + if child_xpath in repeats or (encrypted and child_name == "media"): node_type = list if node_type == dict: @@ -225,7 +231,7 @@ def _flatten_dict(d, prefix): # hack: removing [1] index to be consistent across # surveys that have a single repitition of the # loop versus mutliple. - item_prefix[-1] += u"[%s]" % text(i + 1) + item_prefix[-1] += "[%s]" % text(i + 1) if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): @@ -256,13 +262,12 @@ def _flatten_dict_nest_repeats(d, prefix): if isinstance(item, dict): repeat = {} - for path, value in _flatten_dict_nest_repeats( - item, item_prefix): + for path, value in _flatten_dict_nest_repeats(item, item_prefix): # TODO: this only considers the first level of repeats - repeat.update({u"/".join(path[1:]): value}) + repeat.update({"/".join(path[1:]): value}) repeats.append(repeat) else: - repeats.append({u"/".join(item_prefix[1:]): item}) + repeats.append({"/".join(item_prefix[1:]): item}) yield (new_prefix, repeats) else: yield (new_prefix, value) @@ -300,7 +305,6 @@ def _get_all_attributes(node): class XFormInstanceParser(object): - def __init__(self, xml_str, data_dictionary): self.dd = data_dictionary self.parse(xml_str) @@ -308,18 +312,19 @@ def __init__(self, xml_str, data_dictionary): def parse(self, xml_str): self._xml_obj = clean_and_parse_xml(xml_str) self._root_node = self._xml_obj.documentElement - repeats = [e.get_abbreviated_xpath() - for e in self.dd.get_survey_elements_of_type(u"repeat")] + repeats = [ + e.get_abbreviated_xpath() + for e in self.dd.get_survey_elements_of_type("repeat") + ] - self._dict = _xml_node_to_dict(self._root_node, repeats, - self.dd.encrypted) + self._dict = _xml_node_to_dict(self._root_node, repeats, self.dd.encrypted) self._flat_dict = {} if self._dict is None: raise InstanceEmptyError for path, value in _flatten_dict_nest_repeats(self._dict, []): - self._flat_dict[u"/".join(path[1:])] = value + self._flat_dict["/".join(path[1:])] = value self._set_attributes() def get_root_node(self): @@ -348,17 +353,18 @@ def _set_attributes(self): # multiple xml tags, overriding and log when this occurs if key in self._attributes: logger = logging.getLogger("console_logger") - logger.debug("Skipping duplicate attribute: %s" - " with value %s" % (key, value)) + logger.debug( + "Skipping duplicate attribute: %s" " with value %s" % (key, value) + ) logger.debug(text(all_attributes)) else: self._attributes[key] = value def get_xform_id_string(self): - return self._attributes[u"id"] + return self._attributes["id"] def get_version(self): - return self._attributes.get(u"version") + return self._attributes.get("version") def get_flat_dict_with_attributes(self): result = self.to_flat_dict().copy() diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index b9fa3b8d0c..52a8589d97 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -4,8 +4,8 @@ """ import os import re -from future.moves.urllib.parse import urlparse -from future.moves.urllib.request import urlopen +from six.moves.urllib.parse import urlparse +from six.moves.urllib.request import urlopen import requests from django import forms @@ -28,37 +28,38 @@ from onadata.libs.utils.user_auth import get_user_default_project FORM_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('https://creativecommons.org/licenses/by/3.0/', - ugettext_lazy('Attribution CC BY')), - ('https://creativecommons.org/licenses/by-sa/3.0/', - ugettext_lazy('Attribution-ShareAlike CC BY-SA')), + ("No License", ugettext_lazy("No License")), + ( + "https://creativecommons.org/licenses/by/3.0/", + ugettext_lazy("Attribution CC BY"), + ), + ( + "https://creativecommons.org/licenses/by-sa/3.0/", + ugettext_lazy("Attribution-ShareAlike CC BY-SA"), + ), ) DATA_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('http://opendatacommons.org/licenses/pddl/summary/', - ugettext_lazy('PDDL')), - ('http://opendatacommons.org/licenses/by/summary/', - ugettext_lazy('ODC-BY')), - ('http://opendatacommons.org/licenses/odbl/summary/', - ugettext_lazy('ODBL')), + ("No License", ugettext_lazy("No License")), + ("http://opendatacommons.org/licenses/pddl/summary/", ugettext_lazy("PDDL")), + ("http://opendatacommons.org/licenses/by/summary/", ugettext_lazy("ODC-BY")), + ("http://opendatacommons.org/licenses/odbl/summary/", ugettext_lazy("ODBL")), ) PERM_CHOICES = ( - ('view', ugettext_lazy('Can view')), - ('edit', ugettext_lazy('Can edit')), - ('report', ugettext_lazy('Can submit to')), - ('remove', ugettext_lazy('Remove permissions')), + ("view", ugettext_lazy("Can view")), + ("edit", ugettext_lazy("Can edit")), + ("report", ugettext_lazy("Can submit to")), + ("remove", ugettext_lazy("Remove permissions")), ) VALID_XLSFORM_CONTENT_TYPES = [ - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'text/csv', - 'application/vnd.ms-excel' + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "text/csv", + "application/vnd.ms-excel", ] -VALID_FILE_EXTENSIONS = ['.xls', '.xlsx', '.csv'] +VALID_FILE_EXTENSIONS = [".xls", ".xlsx", ".csv"] def get_filename(response): @@ -70,11 +71,11 @@ def get_filename(response): # following format: # 'attachment; filename="ActApp_Survey_System.xlsx"; filename*=UTF-8\'\'ActApp_Survey_System.xlsx' # noqa cleaned_xls_file = "" - content = response.headers.get('content-disposition').split('; ') - counter = [a for a in content if a.startswith('filename=')] + content = response.headers.get("content-disposition").split("; ") + counter = [a for a in content if a.startswith("filename=")] if len(counter) >= 1: filename_key_val = counter[0] - filename = filename_key_val.split('=')[1].replace("\"", "") + filename = filename_key_val.split("=")[1].replace('"', "") name, extension = os.path.splitext(filename) if extension in VALID_FILE_EXTENSIONS and name: @@ -84,36 +85,40 @@ def get_filename(response): class DataLicenseForm(forms.Form): - """" + """ " Data license form. """ - value = forms.ChoiceField(choices=DATA_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'data-license'})) + + value = forms.ChoiceField( + choices=DATA_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "data-license"}), + ) class FormLicenseForm(forms.Form): """ Form license form. """ - value = forms.ChoiceField(choices=FORM_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'form-license'})) + + value = forms.ChoiceField( + choices=FORM_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "form-license"}), + ) class PermissionForm(forms.Form): """ Permission assignment form. """ + for_user = forms.CharField( widget=forms.TextInput( attrs={ - 'id': 'autocomplete', - 'data-provide': 'typeahead', - 'autocomplete': 'off' - }) + "id": "autocomplete", + "data-provide": "typeahead", + "autocomplete": "off", + } + ) ) perm_type = forms.ChoiceField(choices=PERM_CHOICES, widget=forms.Select()) @@ -129,14 +134,15 @@ class UserProfileForm(ModelForm): class Meta: model = UserProfile - exclude = ('user', 'created_by', 'num_of_submissions') + exclude = ("user", "created_by", "num_of_submissions") + email = forms.EmailField(widget=forms.TextInput()) def clean_metadata(self): """ Returns an empty dict if metadata is None. """ - metadata = self.cleaned_data.get('metadata') + metadata = self.cleaned_data.get("metadata") return metadata if metadata is not None else dict() @@ -145,43 +151,49 @@ class UserProfileFormRegister(forms.Form): """ User profile registration form. """ - first_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - last_name = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - city = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - country = forms.ChoiceField(widget=forms.Select(), required=False, - choices=COUNTRIES, initial='ZZ') - organization = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - home_page = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - twitter = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) + + first_name = forms.CharField( + widget=forms.TextInput(), required=True, max_length=255 + ) + last_name = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + city = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) + country = forms.ChoiceField( + widget=forms.Select(), required=False, choices=COUNTRIES, initial="ZZ" + ) + organization = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + home_page = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + twitter = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) def save_user_profile(self, new_user): """ Creates and returns a new_user profile. """ - new_profile = \ - UserProfile(user=new_user, name=self.cleaned_data['first_name'], - city=self.cleaned_data['city'], - country=self.cleaned_data['country'], - organization=self.cleaned_data['organization'], - home_page=self.cleaned_data['home_page'], - twitter=self.cleaned_data['twitter']) + new_profile = UserProfile( + user=new_user, + name=self.cleaned_data["first_name"], + city=self.cleaned_data["city"], + country=self.cleaned_data["country"], + organization=self.cleaned_data["organization"], + home_page=self.cleaned_data["home_page"], + twitter=self.cleaned_data["twitter"], + ) new_profile.save() return new_profile # pylint: disable=too-many-ancestors # order of inheritance control order of form display -class RegistrationFormUserProfile(RegistrationFormUniqueEmail, - UserProfileFormRegister): +class RegistrationFormUserProfile(RegistrationFormUniqueEmail, UserProfileFormRegister): """ User profile registration form. """ + RESERVED_USERNAMES = settings.RESERVED_USERNAMES username = forms.CharField(widget=forms.TextInput(), max_length=30) email = forms.EmailField(widget=forms.TextInput()) @@ -191,133 +203,149 @@ def clean_username(self): """ Validate a new user username. """ - username = self.cleaned_data['username'].lower() + username = self.cleaned_data["username"].lower() if username in self.RESERVED_USERNAMES: raise forms.ValidationError( - _(u'%s is a reserved name, please choose another') % username) + _("%s is a reserved name, please choose another") % username + ) elif not self.legal_usernames_re.search(username): raise forms.ValidationError( - _(u'username may only contain alpha-numeric characters and ' - u'underscores')) + _( + "username may only contain alpha-numeric characters and " + "underscores" + ) + ) try: User.objects.get(username=username) except User.DoesNotExist: return username - raise forms.ValidationError(_(u'%s already exists') % username) + raise forms.ValidationError(_("%s already exists") % username) class SourceForm(forms.Form): """ Source document form. """ - source = forms.FileField(label=ugettext_lazy(u"Source document"), - required=True) + + source = forms.FileField(label=ugettext_lazy("Source document"), required=True) class SupportDocForm(forms.Form): """ Supporting document. """ - doc = forms.FileField(label=ugettext_lazy(u"Supporting document"), - required=True) + + doc = forms.FileField(label=ugettext_lazy("Supporting document"), required=True) class MediaForm(forms.Form): """ Media file upload form. """ - media = forms.FileField(label=ugettext_lazy(u"Media upload"), - required=True) + + media = forms.FileField(label=ugettext_lazy("Media upload"), required=True) def clean_media(self): """ Validate media upload file. """ - data_type = self.cleaned_data['media'].content_type - if data_type not in ['image/jpeg', 'image/png', 'audio/mpeg']: - raise forms.ValidationError('Only these media types are \ - allowed .png .jpg .mp3 .3gp .wav') + data_type = self.cleaned_data["media"].content_type + if data_type not in ["image/jpeg", "image/png", "audio/mpeg"]: + raise forms.ValidationError( + "Only these media types are \ + allowed .png .jpg .mp3 .3gp .wav" + ) class MapboxLayerForm(forms.Form): """ Mapbox layers form. """ - map_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - attribution = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - link = forms.URLField(label=ugettext_lazy(u'JSONP url'), - required=True) + + map_name = forms.CharField(widget=forms.TextInput(), required=True, max_length=255) + attribution = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + link = forms.URLField(label=ugettext_lazy("JSONP url"), required=True) class QuickConverterFile(forms.Form): """ Uploads XLSForm form. """ - xls_file = forms.FileField( - label=ugettext_lazy(u'XLS File'), required=False) + + xls_file = forms.FileField(label=ugettext_lazy("XLS File"), required=False) class QuickConverterURL(forms.Form): """ Uploads XLSForm from a URL. """ - xls_url = forms.URLField(label=ugettext_lazy('XLS URL'), - required=False) + + xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) class QuickConverterDropboxURL(forms.Form): """ Uploads XLSForm from Dropbox. """ - dropbox_xls_url = forms.URLField( - label=ugettext_lazy('XLS URL'), required=False) + + dropbox_xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) class QuickConverterCsvFile(forms.Form): """ Uploads CSV XLSForm. """ - csv_url = forms.URLField( - label=ugettext_lazy('CSV URL'), required=False) + + csv_url = forms.URLField(label=ugettext_lazy("CSV URL"), required=False) class QuickConverterTextXlsForm(forms.Form): """ Uploads Text XLSForm. """ + text_xls_form = forms.CharField( - label=ugettext_lazy('XLSForm Representation'), required=False) + label=ugettext_lazy("XLSForm Representation"), required=False + ) class QuickConverterXmlFile(forms.Form): """ Uploads an XForm XML. """ - xml_file = forms.FileField( - label=ugettext_lazy(u'XML File'), required=False) + + xml_file = forms.FileField(label=ugettext_lazy("XML File"), required=False) class QuickConverterFloipFile(forms.Form): """ Uploads a FLOIP results data package descriptor file. """ + floip_file = forms.FileField( - label=ugettext_lazy(u'FlOIP results data packages descriptor File'), - required=False) + label=ugettext_lazy("FlOIP results data packages descriptor File"), + required=False, + ) # pylint: disable=too-many-ancestors -class QuickConverter(QuickConverterFile, QuickConverterURL, - QuickConverterDropboxURL, QuickConverterTextXlsForm, - QuickConverterCsvFile, QuickConverterXmlFile, - QuickConverterFloipFile): +class QuickConverter( + QuickConverterFile, + QuickConverterURL, + QuickConverterDropboxURL, + QuickConverterTextXlsForm, + QuickConverterCsvFile, + QuickConverterXmlFile, + QuickConverterFloipFile, +): """ Publish XLSForm and convert to XForm. """ + project = forms.IntegerField(required=False) validate = URLValidator() @@ -325,14 +353,13 @@ def clean_project(self): """ Project validation. """ - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is not None: try: # pylint: disable=attribute-defined-outside-init, no-member self._project = Project.objects.get(pk=int(project)) except (Project.DoesNotExist, ValueError): - raise forms.ValidationError( - _(u"Unknown project id: %s" % project)) + raise forms.ValidationError(_("Unknown project id: %s" % project)) return project @@ -345,95 +372,105 @@ def publish(self, user, id_string=None, created_by=None): # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None - if 'text_xls_form' in self.cleaned_data\ - and self.cleaned_data['text_xls_form'].strip(): - csv_data = self.cleaned_data['text_xls_form'] + if ( + "text_xls_form" in self.cleaned_data + and self.cleaned_data["text_xls_form"].strip() + ): + csv_data = self.cleaned_data["text_xls_form"] # assigning the filename to a random string (quick fix) import random - rand_name = "uploaded_form_%s.csv" % ''.join( - random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) - - cleaned_xls_file = \ - default_storage.save( - upload_to(None, rand_name, user.username), - ContentFile(csv_data.encode())) - if 'xls_file' in self.cleaned_data and\ - self.cleaned_data['xls_file']: - cleaned_xls_file = self.cleaned_data['xls_file'] - if 'floip_file' in self.cleaned_data and\ - self.cleaned_data['floip_file']: - cleaned_xls_file = self.cleaned_data['floip_file'] + + rand_name = "uploaded_form_%s.csv" % "".join( + random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) + ) + + cleaned_xls_file = default_storage.save( + upload_to(None, rand_name, user.username), + ContentFile(csv_data.encode()), + ) + if "xls_file" in self.cleaned_data and self.cleaned_data["xls_file"]: + cleaned_xls_file = self.cleaned_data["xls_file"] + if "floip_file" in self.cleaned_data and self.cleaned_data["floip_file"]: + cleaned_xls_file = self.cleaned_data["floip_file"] cleaned_url = ( - self.cleaned_data['xls_url'].strip() or - self.cleaned_data['dropbox_xls_url'] or - self.cleaned_data['csv_url']) + self.cleaned_data["xls_url"].strip() + or self.cleaned_data["dropbox_xls_url"] + or self.cleaned_data["csv_url"] + ) if cleaned_url: cleaned_xls_file = urlparse(cleaned_url) - cleaned_xls_file = \ - '_'.join(cleaned_xls_file.path.split('/')[-2:]) + cleaned_xls_file = "_".join(cleaned_xls_file.path.split("/")[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: response = requests.get(cleaned_url) - if response.headers.get('content-type') in \ - VALID_XLSFORM_CONTENT_TYPES and \ - response.status_code < 400: + if ( + response.headers.get("content-type") + in VALID_XLSFORM_CONTENT_TYPES + and response.status_code < 400 + ): cleaned_xls_file = get_filename(response) - cleaned_xls_file = \ - upload_to(None, cleaned_xls_file, user.username) + cleaned_xls_file = upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urlopen(cleaned_url).read()) - cleaned_xls_file = \ - default_storage.save(cleaned_xls_file, xls_data) + cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is None: project = get_user_default_project(user) else: project = self._project - cleaned_xml_file = self.cleaned_data['xml_file'] + cleaned_xml_file = self.cleaned_data["xml_file"] if cleaned_xml_file: - return publish_xml_form(cleaned_xml_file, user, project, - id_string, created_by or user) + return publish_xml_form( + cleaned_xml_file, user, project, id_string, created_by or user + ) if cleaned_xls_file is None: raise forms.ValidationError( - _(u"XLSForm not provided, expecting either of these" - " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," - " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) + _( + "XLSForm not provided, expecting either of these" + " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," + " 'dropbox_xls_url', 'text_xls_form', 'floip_file'" + ) + ) # publish the xls - return publish_xls_form(cleaned_xls_file, user, project, - id_string, created_by or user) + return publish_xls_form( + cleaned_xls_file, user, project, id_string, created_by or user + ) class ActivateSMSSupportForm(forms.Form): """ Enable SMS support form. """ - enable_sms_support = forms.TypedChoiceField(coerce=lambda x: x == 'True', - choices=((False, 'No'), - (True, 'Yes')), - widget=forms.Select, - label=ugettext_lazy( - u"Enable SMS Support")) - sms_id_string = forms.CharField(max_length=50, required=True, - label=ugettext_lazy(u"SMS Keyword")) + + enable_sms_support = forms.TypedChoiceField( + coerce=lambda x: x == "True", + choices=((False, "No"), (True, "Yes")), + widget=forms.Select, + label=ugettext_lazy("Enable SMS Support"), + ) + sms_id_string = forms.CharField( + max_length=50, required=True, label=ugettext_lazy("SMS Keyword") + ) def clean_sms_id_string(self): """ SMS id_string validation. """ - sms_id_string = self.cleaned_data.get('sms_id_string', '').strip() + sms_id_string = self.cleaned_data.get("sms_id_string", "").strip() - if not re.match(r'^[a-z0-9\_\-]+$', sms_id_string): - raise forms.ValidationError(u"id_string can only contain alphanum" - u" characters") + if not re.match(r"^[a-z0-9\_\-]+$", sms_id_string): + raise forms.ValidationError( + "id_string can only contain alphanum" " characters" + ) return sms_id_string @@ -442,8 +479,9 @@ class ExternalExportForm(forms.Form): """ XLS reports form. """ - template_name = forms.CharField(label='Template Name', max_length=20) - template_token = forms.URLField(label='Template URL', max_length=100) + + template_name = forms.CharField(label="Template Name", max_length=20) + template_token = forms.URLField(label="Template URL", max_length=100) # Deprecated diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 691a2d1a8c..117d118b0f 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -9,17 +9,20 @@ from django.db import models from django.db.models.signals import post_save, pre_save from django.utils.translation import ugettext_lazy -from django.utils.encoding import python_2_unicode_compatible from guardian.shortcuts import get_perms_for_model, assign_perm from guardian.models import UserObjectPermissionBase from guardian.models import GroupObjectPermissionBase from rest_framework.authtoken.models import Token +from six import python_2_unicode_compatible from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists from onadata.apps.main.signals import ( - set_api_permissions, send_inactive_user_email, send_activation_email) + set_api_permissions, + send_inactive_user_email, + send_activation_email, +) -REQUIRE_AUTHENTICATION = 'REQUIRE_ODK_AUTHENTICATION' +REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" @python_2_unicode_compatible @@ -29,8 +32,7 @@ class UserProfile(models.Model): """ # This field is required. - user = models.OneToOneField( - User, related_name='profile', on_delete=models.CASCADE) + user = models.OneToOneField(User, related_name="profile", on_delete=models.CASCADE) # Other fields here name = models.CharField(max_length=255, blank=True) @@ -41,18 +43,19 @@ class UserProfile(models.Model): twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( - default=False, - verbose_name=ugettext_lazy("Require Phone Authentication")) + default=False, verbose_name=ugettext_lazy("Require Phone Authentication") + ) address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) num_of_submissions = models.IntegerField(default=0) metadata = JSONField(default=dict, blank=True) date_modified = models.DateTimeField(auto_now=True) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return "%s[%s]" % (self.name, self.user.username) @property def gravatar(self): @@ -68,22 +71,22 @@ def twitter_clean(self): return self.twitter[1:] return self.twitter - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): + def save( + self, force_insert=False, force_update=False, using=None, update_fields=None + ): # Override default save method to set settings configured require_auth # value if self.pk is None and hasattr(settings, REQUIRE_AUTHENTICATION): self.require_auth = getattr(settings, REQUIRE_AUTHENTICATION) - super(UserProfile, self).save(force_insert, force_update, using, - update_fields) + super(UserProfile, self).save(force_insert, force_update, using, update_fields) class Meta: - app_label = 'main' + app_label = "main" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to user profile"), - ('view_profile', "Can view user profile"), + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to user profile"), + ("view_profile", "Can view user profile"), ) @@ -101,47 +104,46 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -def set_kpi_formbuilder_permissions( - sender, instance=None, created=False, **kwargs): +def set_kpi_formbuilder_permissions(sender, instance=None, created=False, **kwargs): if created: - kpi_formbuilder_url = hasattr(settings, 'KPI_FORMBUILDER_URL') and\ - settings.KPI_FORMBUILDER_URL + kpi_formbuilder_url = ( + hasattr(settings, "KPI_FORMBUILDER_URL") and settings.KPI_FORMBUILDER_URL + ) if kpi_formbuilder_url: requests.post( - "%s/%s" % ( - kpi_formbuilder_url, - 'grant-default-model-level-perms' - ), - headers={ - 'Authorization': 'Token %s' % instance.user.auth_token - } + "%s/%s" % (kpi_formbuilder_url, "grant-default-model-level-perms"), + headers={"Authorization": "Token %s" % instance.user.auth_token}, ) -post_save.connect(create_auth_token, sender=User, dispatch_uid='auth_token') +post_save.connect(create_auth_token, sender=User, dispatch_uid="auth_token") post_save.connect( - send_inactive_user_email, sender=User, - dispatch_uid='send_inactive_user_email') + send_inactive_user_email, sender=User, dispatch_uid="send_inactive_user_email" +) pre_save.connect( - send_activation_email, sender=User, - dispatch_uid='send_activation_email' + send_activation_email, sender=User, dispatch_uid="send_activation_email" ) -post_save.connect(set_api_permissions, sender=User, - dispatch_uid='set_api_permissions') +post_save.connect(set_api_permissions, sender=User, dispatch_uid="set_api_permissions") -post_save.connect(set_object_permissions, sender=UserProfile, - dispatch_uid='set_object_permissions') +post_save.connect( + set_object_permissions, sender=UserProfile, dispatch_uid="set_object_permissions" +) -post_save.connect(set_kpi_formbuilder_permissions, sender=UserProfile, - dispatch_uid='set_kpi_formbuilder_permission') +post_save.connect( + set_kpi_formbuilder_permissions, + sender=UserProfile, + dispatch_uid="set_kpi_formbuilder_permission", +) class UserProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) class UserProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index d580c353fc..c463165821 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -6,8 +6,8 @@ import re import socket from builtins import open -from future.moves.urllib.error import URLError -from future.moves.urllib.request import urlopen +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen from io import StringIO from tempfile import NamedTemporaryFile @@ -29,27 +29,31 @@ from onadata.apps.logger.views import submission from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models import DataDictionary -from onadata.libs.utils.common_tools import (filename_from_disposition, - get_response_content) +from onadata.libs.utils.common_tools import ( + filename_from_disposition, + get_response_content, +) from onadata.libs.utils.user_auth import get_user_default_project class TestBase(PyxformMarkdown, TransactionTestCase): maxDiff = None - surveys = ['transport_2011-07-25_19-05-49', - 'transport_2011-07-25_19-05-36', - 'transport_2011-07-25_19-06-01', - 'transport_2011-07-25_19-06-14'] + surveys = [ + "transport_2011-07-25_19-05-49", + "transport_2011-07-25_19-05-36", + "transport_2011-07-25_19-06-01", + "transport_2011-07-25_19-06-14", + ] this_directory = os.path.abspath(os.path.dirname(__file__)) def setUp(self): self.maxDiff = None self._create_user_and_login() - self.base_url = 'http://testserver' + self.base_url = "http://testserver" self.factory = RequestFactory() def _fixture_path(self, *args): - return os.path.join(os.path.dirname(__file__), 'fixtures', *args) + return os.path.join(os.path.dirname(__file__), "fixtures", *args) def _create_user(self, username, password, create_profile=False): user, created = User.objects.get_or_create(username=username) @@ -74,8 +78,7 @@ def _logout(self, client=None): client = self.client client.logout() - def _create_user_and_login(self, username="bob", password="bob", - factory=None): + def _create_user_and_login(self, username="bob", password="bob", factory=None): self.login_username = username self.login_password = password self.user = self._create_user(username, password, create_profile=True) @@ -85,27 +88,26 @@ def _create_user_and_login(self, username="bob", password="bob", self.anon = Client() def _publish_xls_file(self, path): - if not path.startswith('/%s/' % self.user.username): + if not path.startswith("/%s/" % self.user.username): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as f: + with open(path, "rb") as f: xls_file = InMemoryUploadedFile( f, - 'xls_file', + "xls_file", os.path.abspath(os.path.basename(path)), - 'application/vnd.ms-excel', + "application/vnd.ms-excel", os.path.getsize(path), - None) - if not hasattr(self, 'project'): + None, + ) + if not hasattr(self, "project"): self.project = get_user_default_project(self.user) DataDictionary.objects.create( - created_by=self.user, - user=self.user, - xls=xls_file, - project=self.project) + created_by=self.user, user=self.user, xls=xls_file, project=self.project + ) def _publish_xlsx_file(self): - path = os.path.join(self.this_directory, 'fixtures', 'exp.xlsx') + path = os.path.join(self.this_directory, "fixtures", "exp.xlsx") pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) # make sure publishing the survey worked @@ -115,41 +117,68 @@ def _publish_xls_file_and_set_xform(self, path): count = XForm.objects.count() self._publish_xls_file(path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] - def _share_form_data(self, id_string='transportation_2011_07_25'): + def _share_form_data(self, id_string="transportation_2011_07_25"): xform = XForm.objects.get(id_string=id_string) xform.shared_data = True xform.save() def _publish_transportation_form(self): xls_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xls") + self.this_directory, "fixtures", "transportation", "transportation.xls" + ) count = XForm.objects.count() TestBase._publish_xls_file(self, xls_path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] def _submit_transport_instance(self, survey_at=0): s = self.surveys[survey_at] - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + ) def _submit_transport_instance_w_uuid(self, name): - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances_w_uuid', name, name + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances_w_uuid", + name, + name + ".xml", + ) + ) def _submit_transport_instance_w_attachment(self, survey_at=0): s = self.surveys[survey_at] media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', s, media_file)) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ), + ) self.attachment = Attachment.objects.all().reverse()[0] self.attachment_media_file = self.attachment.media_file @@ -158,48 +187,54 @@ def _publish_transportation_form_and_submit_instance(self): self._submit_transport_instance() def _make_submissions_gps(self): - surveys = ['gps_1980-01-23_20-52-08', - 'gps_1980-01-23_21-21-33', ] + surveys = [ + "gps_1980-01-23_20-52-08", + "gps_1980-01-23_21-21-33", + ] for survey in surveys: - path = self._fixture_path('gps', 'instances', survey + '.xml') + path = self._fixture_path("gps", "instances", survey + ".xml") self._make_submission(path) - def _make_submission(self, path, username=None, add_uuid=False, - forced_submission_time=None, auth=None, client=None): + def _make_submission( + self, + path, + username=None, + add_uuid=False, + forced_submission_time=None, + auth=None, + client=None, + ): # store temporary file with dynamic uuid self.factory = APIRequestFactory() if auth is None: - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") tmp_file = None if add_uuid: - tmp_file = NamedTemporaryFile(delete=False, mode='w') + tmp_file = NamedTemporaryFile(delete=False, mode="w") split_xml = None - with open(path, encoding='utf-8') as _file: - split_xml = re.split(r'()', _file.read()) + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) - split_xml[1:1] = [ - '%s' % self.xform.uuid - ] - tmp_file.write(''.join(split_xml)) + split_xml[1:1] = ["%s" % self.xform.uuid] + tmp_file.write("".join(split_xml)) path = tmp_file.name tmp_file.close() - with open(path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} if username is None: username = self.user.username - url_prefix = '%s/' % username if username else '' - url = '/%ssubmission' % url_prefix + url_prefix = "%s/" % username if username else "" + url = "/%ssubmission" % url_prefix request = self.factory.post(url, post_data) - request.user = authenticate(username=auth.username, - password=auth.password) + request.user = authenticate(username=auth.username, password=auth.password) self.response = submission(request, username=username) @@ -208,7 +243,7 @@ def _make_submission(self, path, username=None, add_uuid=False, self.response = submission(request, username=username) if forced_submission_time: - instance = Instance.objects.order_by('-pk').all()[0] + instance = Instance.objects.order_by("-pk").all()[0] instance.date_created = forced_submission_time instance.json = instance.get_full_dict() instance.save() @@ -219,33 +254,27 @@ def _make_submission(self, path, username=None, add_uuid=False, os.unlink(tmp_file.name) def _make_submission_w_attachment(self, path, attachment_path): - with open(path, encoding='utf-8') as f: - data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + data = {"xml_submission_file": f} if attachment_path is not None: if isinstance(attachment_path, list): for c in range(len(attachment_path)): - data['media_file_{}'.format(c)] = open( - attachment_path[c], 'rb') + data["media_file_{}".format(c)] = open(attachment_path[c], "rb") else: - data['media_file'] = open( - attachment_path, 'rb') + data["media_file"] = open(attachment_path, "rb") - url = '/%s/submission' % self.user.username - auth = DigestAuth('bob', 'bob') + url = "/%s/submission" % self.user.username + auth = DigestAuth("bob", "bob") self.factory = APIRequestFactory() request = self.factory.post(url, data) - request.user = authenticate(username='bob', - password='bob') - self.response = submission(request, - username=self.user.username) + request.user = authenticate(username="bob", password="bob") + self.response = submission(request, username=self.user.username) if auth and self.response.status_code == 401: request.META.update(auth(request.META, self.response)) - self.response = submission(request, - username=self.user.username) + self.response = submission(request, username=self.user.username) - def _make_submissions(self, username=None, add_uuid=False, - should_store=True): + def _make_submissions(self, username=None, add_uuid=False, should_store=True): """Make test fixture submissions to current xform. :param username: submit under this username, default None. @@ -253,16 +282,23 @@ def _make_submissions(self, username=None, add_uuid=False, :param should_store: should submissions be save, default True. """ - paths = [os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', s, s + '.xml') for s in self.surveys] + paths = [ + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + for s in self.surveys + ] pre_count = Instance.objects.count() for path in paths: self._make_submission(path, username, add_uuid) - post_count = pre_count + len(self.surveys) if should_store\ - else pre_count + post_count = pre_count + len(self.surveys) if should_store else pre_count self.assertEqual(Instance.objects.count(), post_count) self.assertEqual(self.xform.instances.count(), post_count) xform = XForm.objects.get(pk=self.xform.pk) @@ -277,26 +313,25 @@ def _check_url(self, url, timeout=1): pass return False - def _internet_on(self, url='http://74.125.113.99'): + def _internet_on(self, url="http://74.125.113.99"): # default value is some google IP return self._check_url(url) def _set_auth_headers(self, username, password): return { - 'HTTP_AUTHORIZATION': - 'Basic ' + base64.b64encode(( - '%s:%s' % (username, password)).encode( - 'utf-8')).decode('utf-8'), + "HTTP_AUTHORIZATION": "Basic " + + base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode( + "utf-8" + ), } - def _get_authenticated_client( - self, url, username='bob', password='bob', extra={}): + def _get_authenticated_client(self, url, username="bob", password="bob", extra={}): client = DigestClient() # request with no credentials req = client.get(url, {}, **extra) self.assertEqual(req.status_code, 401) # apply credentials - client.set_authorization(username, password, 'Digest') + client.set_authorization(username, password, "Digest") return client def _set_mock_time(self, mock_time): @@ -311,24 +346,37 @@ def _set_require_auth(self, auth=True): def _get_digest_client(self): self._set_require_auth(True) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") return client def _publish_submit_geojson(self): path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "geolocation", "GeoLocationForm.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm.xlsx", + ) self._publish_xls_file_and_set_xform(path) - view = XFormViewSet.as_view({'post': 'csv_import'}) - csv_import = \ - open(os.path.join(settings.PROJECT_ROOT, 'apps', 'main', - 'tests', 'fixtures', 'geolocation', - 'GeoLocationForm_2015_01_15_01_28_45.csv'), - encoding='utf-8') - post_data = {'csv_file': csv_import} - request = self.factory.post('/', data=post_data, **self.extra) + view = XFormViewSet.as_view({"post": "csv_import"}) + csv_import = open( + os.path.join( + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm_2015_01_15_01_28_45.csv", + ), + encoding="utf-8", + ) + post_data = {"csv_file": csv_import} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) @@ -336,30 +384,34 @@ def _publish_markdown(self, md_xlsform, user, project=None, **kwargs): """ Publishes a markdown XLSForm. """ - kwargs['name'] = 'data' + kwargs["name"] = "data" survey = self.md_to_pyxform_survey(md_xlsform, kwargs=kwargs) - survey['sms_keyword'] = survey['id_string'] - if not project or not hasattr(self, 'project'): + survey["sms_keyword"] = survey["id_string"] + if not project or not hasattr(self, "project"): project = get_user_default_project(user) - xform = DataDictionary(created_by=user, user=user, - xml=survey.to_xml(), json=survey.to_json(), - project=project) + xform = DataDictionary( + created_by=user, + user=user, + xml=survey.to_xml(), + json=survey.to_json(), + project=project, + ) xform.save() return xform def _test_csv_response(self, response, csv_file_path): headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) __, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") data = get_response_content(response) reader = csv.DictReader(StringIO(data)) data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: @@ -369,7 +421,7 @@ def _test_csv_response(self, response, csv_file_path): def _test_csv_files(self, csv_file, csv_file_path): reader = csv.DictReader(csv_file) data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: diff --git a/onadata/apps/main/tests/test_form_enter_data.py b/onadata/apps/main/tests/test_form_enter_data.py index 27b1ebca23..0eb2b18186 100644 --- a/onadata/apps/main/tests/test_form_enter_data.py +++ b/onadata/apps/main/tests/test_form_enter_data.py @@ -7,7 +7,7 @@ from django.core.validators import URLValidator from django.test import RequestFactory from django.urls import reverse -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from httmock import HTTMock, urlmatch from nose import SkipTest diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index dea707cd87..c658dc1e09 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -14,7 +14,7 @@ from django.core.files.uploadedfile import UploadedFile from django.urls import reverse from django_digest.test import Client as DigestClient -from future.utils import iteritems +from six import iteritems from mock import patch from xlrd import open_workbook @@ -27,24 +27,23 @@ from onadata.libs.utils.common_tags import MONGO_STRFTIME from onadata.libs.utils.common_tools import get_response_content -uuid_regex = re.compile( - r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) +uuid_regex = re.compile(r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) class TestProcess(TestBase): - loop_str = 'loop_over_transport_types_frequency' - frequency_str = 'frequency_to_referral_facility' - ambulance_key = '%s/ambulance/%s' % (loop_str, frequency_str) - bicycle_key = '%s/bicycle/%s' % (loop_str, frequency_str) - other_key = '%s/other/%s' % (loop_str, frequency_str) - taxi_key = '%s/taxi/%s' % (loop_str, frequency_str) - transport_ambulance_key = u'transport/%s' % ambulance_key - transport_bicycle_key = u'transport/%s' % bicycle_key + loop_str = "loop_over_transport_types_frequency" + frequency_str = "frequency_to_referral_facility" + ambulance_key = "%s/ambulance/%s" % (loop_str, frequency_str) + bicycle_key = "%s/bicycle/%s" % (loop_str, frequency_str) + other_key = "%s/other/%s" % (loop_str, frequency_str) + taxi_key = "%s/taxi/%s" % (loop_str, frequency_str) + transport_ambulance_key = "transport/%s" % ambulance_key + transport_bicycle_key = "transport/%s" % bicycle_key uuid_to_submission_times = { - '5b2cc313-fc09-437e-8149-fcd32f695d41': '2013-02-14T15:37:21', - 'f3d8dc65-91a6-4d0f-9e97-802128083390': '2013-02-14T15:37:22', - '9c6f3468-cfda-46e8-84c1-75458e72805d': '2013-02-14T15:37:23', - '9f0a1508-c3b7-4c99-be00-9b237c26bcbf': '2013-02-14T15:37:24' + "5b2cc313-fc09-437e-8149-fcd32f695d41": "2013-02-14T15:37:21", + "f3d8dc65-91a6-4d0f-9e97-802128083390": "2013-02-14T15:37:22", + "9c6f3468-cfda-46e8-84c1-75458e72805d": "2013-02-14T15:37:23", + "9f0a1508-c3b7-4c99-be00-9b237c26bcbf": "2013-02-14T15:37:24", } def setUp(self): @@ -66,54 +65,71 @@ def _update_dynamic_data(self): """ Update stuff like submission time so we can compare within out fixtures """ - for (uuid, submission_time) in iteritems( - self.uuid_to_submission_times): + for (uuid, submission_time) in iteritems(self.uuid_to_submission_times): i = self.xform.instances.get(uuid=uuid) - i.date_created = pytz.timezone('UTC').localize( - datetime.strptime(submission_time, MONGO_STRFTIME)) + i.date_created = pytz.timezone("UTC").localize( + datetime.strptime(submission_time, MONGO_STRFTIME) + ) i.json = i.get_full_dict() i.save() def test_uuid_submit(self): self._publish_xls_file() - survey = 'transport_2011-07-25_19-05-49' + survey = "transport_2011-07-25_19-05-49" path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml') + self.this_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) with open(path) as f: - post_data = {'xml_submission_file': f, 'uuid': self.xform.uuid} - url = '/submission' + post_data = {"xml_submission_file": f, "uuid": self.xform.uuid} + url = "/submission" self.response = self.client.post(url, post_data) def test_publish_xlsx_file(self): self._publish_xlsx_file() - @patch('onadata.apps.main.forms.requests') - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.requests") + @patch("onadata.apps.main.forms.urlopen") def test_google_url_upload(self, mock_urlopen, mock_requests): if self._internet_on(url="http://google.com"): - xls_url = "https://docs.google.com/spreadsheet/pub?"\ + xls_url = ( + "https://docs.google.com/spreadsheet/pub?" "key=0AvhZpT7ZLAWmdDhISGhqSjBOSl9XdXd5SHZHUUE2RFE&output=xls" + ) pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xls") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xls", + ) + + xls_file = open(path, "rb") mock_response = requests.Response() mock_response.status_code = 200 mock_response.headers = { - 'content-type': ("application/vnd.openxmlformats-" - "officedocument.spreadsheetml.sheet"), - 'content-disposition': ( + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "content-disposition": ( 'attachment; filename="transportation.' - 'xls"; filename*=UTF-8\'\'transportation.xls') + "xls\"; filename*=UTF-8''transportation.xls" + ), } mock_requests.get.return_value = mock_response mock_urlopen.return_value = xls_file - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) mock_urlopen.assert_called_with(xls_url) mock_requests.get.assert_called_with(xls_url) @@ -123,21 +139,28 @@ def test_google_url_upload(self, mock_urlopen, mock_requests): self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.urlopen") def test_url_upload(self, mock_urlopen): if self._internet_on(url="http://google.com"): - xls_url = 'https://ona.io/examples/forms/tutorial/form.xls' + xls_url = "https://ona.io/examples/forms/tutorial/form.xls" pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xls") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xls", + ) + + xls_file = open(path, "rb") mock_urlopen.return_value = xls_file - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) mock_urlopen.assert_called_with(xls_url) # cleanup the resources @@ -148,10 +171,9 @@ def test_url_upload(self, mock_urlopen): self.assertEqual(XForm.objects.count(), pre_count + 1) def test_bad_url_upload(self): - xls_url = 'formhuborg/pld/forms/transportation_2011_07_25/form.xls' + xls_url = "formhuborg/pld/forms/transportation_2011_07_25/form.xls" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post("/%s/" % self.user.username, {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count) @@ -166,35 +188,36 @@ def test_upload_all_xls(self): success = True for root, sub_folders, filenames in os.walk(root_dir): # ignore files that don't end in '.xls' - for filename in fnmatch.filter(filenames, '*.xls'): - success = self._publish_file(os.path.join(root, filename), - False) + for filename in fnmatch.filter(filenames, "*.xls"): + success = self._publish_file(os.path.join(root, filename), False) if success: # delete it so we don't have id_string conflicts if self.xform: self.xform.delete() self.xform = None - print('finished sub-folder %s' % root) + print("finished sub-folder %s" % root) self.assertEqual(success, True) def test_url_upload_non_dot_xls_path(self): if self._internet_on(): - xls_url = 'http://formhub.org/formhub_u/forms/tutorial/form.xls' + xls_url = "http://formhub.org/formhub_u/forms/tutorial/form.xls" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) def test_not_logged_in_cannot_upload(self): - path = os.path.join(self.this_directory, "fixtures", "transportation", - "transportation.xls") - if not path.startswith('/%s/' % self.user.username): + path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xls" + ) + if not path.startswith("/%s/" % self.user.username): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - return self.client.post('/%s/' % self.user.username, post_data) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + return self.client.post("/%s/" % self.user.username, post_data) def _publish_file(self, xls_path, strict=True): """ @@ -204,7 +227,7 @@ def _publish_file(self, xls_path, strict=True): TestBase._publish_xls_file(self, xls_path) # make sure publishing the survey worked if XForm.objects.count() != pre_count + 1: - print('\nPublish Failure for file: %s' % xls_path) + print("\nPublish Failure for file: %s" % xls_path) if strict: self.assertEqual(XForm.objects.count(), pre_count + 1) else: @@ -213,58 +236,62 @@ def _publish_file(self, xls_path, strict=True): return True def _publish_xls_file(self): - xls_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xls") + xls_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xls" + ) self._publish_file(xls_path) self.assertEqual(self.xform.id_string, "transportation_2011_07_25") def _check_formlist(self): - url = '/%s/formList' % self.user.username + url = "/%s/formList" % self.user.username client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(url) - self.download_url = \ - 'http://testserver/%s/forms/%s/form.xml'\ - % (self.user.username, self.xform.pk) - md5_hash = md5(self.xform.xml.encode('utf-8')).hexdigest() + self.download_url = "http://testserver/%s/forms/%s/form.xml" % ( + self.user.username, + self.xform.pk, + ) + md5_hash = md5(self.xform.xml.encode("utf-8")).hexdigest() expected_content = """ transportation_2011_07_25transportation_2011_07_252014111md5:%(hash)s%(download_url)s""" # noqa expected_content = expected_content % { - 'download_url': self.download_url, - 'hash': md5_hash + "download_url": self.download_url, + "hash": md5_hash, } - self.assertEqual(response.content.decode('utf-8'), expected_content) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue(response.has_header('Date')) + self.assertEqual(response.content.decode("utf-8"), expected_content) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("Date")) def _download_xform(self): client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(self.download_url) response_doc = minidom.parseString(response.content) - xml_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xml") - with open(xml_path, 'rb') as xml_file: + xml_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xml" + ) + with open(xml_path, "rb") as xml_file: expected_doc = minidom.parse(xml_file) model_node = [ - n for n in - response_doc.getElementsByTagName("h:head")[0].childNodes - if n.nodeType == Node.ELEMENT_NODE and - n.tagName == "model"][0] + n + for n in response_doc.getElementsByTagName("h:head")[0].childNodes + if n.nodeType == Node.ELEMENT_NODE and n.tagName == "model" + ][0] # check for UUID and remove - uuid_nodes = [node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + uuid_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(uuid_nodes), 1) uuid_node = uuid_nodes[0] uuid_node.setAttribute("calculate", "''") - response_xml = response_doc.toxml().replace( - self.xform.version, u"201411120717") + response_xml = response_doc.toxml().replace(self.xform.version, "201411120717") # check content without UUID self.assertEqual(response_xml, expected_doc.toxml()) @@ -280,90 +307,98 @@ def _check_data_dictionary(self): qs = DataDictionary.objects.filter(user=self.user) self.assertEqual(qs.count(), 1) self.data_dictionary = DataDictionary.objects.all()[0] - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers.json" + ) + ) as f: expected_list = json.load(f) self.assertEqual(self.data_dictionary.get_headers(), expected_list) # test to make sure the headers in the actual csv are as expected actual_csv = self._get_csv_() - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers_csv.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers_csv.json" + ) + ) as f: expected_list = json.load(f) self.assertEqual(sorted(next(actual_csv)), sorted(expected_list)) def _check_data_for_csv_export(self): data = [ - {"available_transportation_types_to_referral_facility/ambulance": - True, - "available_transportation_types_to_referral_facility/bicycle": - True, - self.ambulance_key: "daily", - self.bicycle_key: "weekly" - }, + { + "available_transportation_types_to_referral_facility/ambulance": True, + "available_transportation_types_to_referral_facility/bicycle": True, + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + }, {}, - {"available_transportation_types_to_referral_facility/ambulance": - True, - self.ambulance_key: "weekly", - }, - {"available_transportation_types_to_referral_facility/taxi": True, - "available_transportation_types_to_referral_facility/other": True, - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - self.other_key: "other", - } + { + "available_transportation_types_to_referral_facility/ambulance": True, + self.ambulance_key: "weekly", + }, + { + "available_transportation_types_to_referral_facility/taxi": True, + "available_transportation_types_to_referral_facility/other": True, + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + self.other_key: "other", + }, ] for d_from_db in self.data_dictionary.get_data_for_excel(): test_dict = {} for (k, v) in iteritems(d_from_db): - if (k not in [u'_xform_id_string', u'meta/instanceID', - '_version', '_id', 'image1']) and v: - new_key = k[len('transport/'):] + if ( + k + not in [ + "_xform_id_string", + "meta/instanceID", + "_version", + "_id", + "image1", + ] + ) and v: + new_key = k[len("transport/") :] test_dict[new_key] = d_from_db[k] self.assertTrue(test_dict in data, (test_dict, data)) data.remove(test_dict) self.assertEquals(data, []) def _check_group_xpaths_do_not_appear_in_dicts_for_export(self): - uuid = u'uuid:f3d8dc65-91a6-4d0f-9e97-802128083390' - instance = self.xform.instances.get(uuid=uuid.split(':')[1]) + uuid = "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390" + instance = self.xform.instances.get(uuid=uuid.split(":")[1]) expected_dict = { - u"transportation": { - u"meta": { - u"instanceID": uuid - }, - u"transport": { - u"loop_over_transport_types_frequency": {u"bicycle": { - u"frequency_to_referral_facility": u"weekly" + "transportation": { + "meta": {"instanceID": uuid}, + "transport": { + "loop_over_transport_types_frequency": { + "bicycle": {"frequency_to_referral_facility": "weekly"}, + "ambulance": {"frequency_to_referral_facility": "daily"}, }, - u"ambulance": { - u"frequency_to_referral_facility": u"daily" - } - }, - u"available_transportation_types_to_referral_facility": - u"ambulance bicycle", - } + "available_transportation_types_to_referral_facility": "ambulance bicycle", + }, } } self.assertEqual(instance.get_dict(flat=False), expected_dict) expected_dict = { - u"transport/available_transportation_types_to_referral_facility": - u"ambulance bicycle", - self.transport_ambulance_key: u"daily", - self.transport_bicycle_key: u"weekly", - u"_xform_id_string": u"transportation_2011_07_25", - u"_version": u"2014111", - u"meta/instanceID": uuid + "transport/available_transportation_types_to_referral_facility": "ambulance bicycle", + self.transport_ambulance_key: "daily", + self.transport_bicycle_key: "weekly", + "_xform_id_string": "transportation_2011_07_25", + "_version": "2014111", + "meta/instanceID": uuid, } self.assertEqual(instance.get_dict(), expected_dict) def _get_csv_(self): # todo: get the csv.reader to handle unicode as done here: # http://docs.python.org/library/csv.html#examples - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -371,18 +406,22 @@ def _get_csv_(self): return csv.reader(actual_lines) def _check_csv_export_first_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) test_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.csv") + self.this_directory, "fixtures", "transportation", "transportation.csv" + ) self._test_csv_response(response, test_file_path) def _check_csv_export_second_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -390,55 +429,72 @@ def _check_csv_export_second_pass(self): actual_csv = csv.reader(actual_lines) headers = next(actual_csv) data = [ - {"image1": "1335783522563.jpg", - 'meta/instanceID': 'uuid:5b2cc313-fc09-437e-8149-fcd32f695d41', - '_uuid': '5b2cc313-fc09-437e-8149-fcd32f695d41', - '_submission_time': '2013-02-14T15:37:21', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '1', '_media_count': '0', - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - "available_transportation_types_to_referral_facility/bicycle": - "True", - self.ambulance_key: "daily", - self.bicycle_key: "weekly", - "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", - '_uuid': 'f3d8dc65-91a6-4d0f-9e97-802128083390', - '_submission_time': '2013-02-14T15:37:22', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - self.ambulance_key: "weekly", - "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", - '_uuid': '9c6f3468-cfda-46e8-84c1-75458e72805d', - '_submission_time': '2013-02-14T15:37:23', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/taxi": - "True", - "available_transportation_types_to_referral_facility/other": - "True", - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", - '_uuid': '9f0a1508-c3b7-4c99-be00-9b237c26bcbf', - '_submission_time': '2013-02-14T15:37:24', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - } + { + "image1": "1335783522563.jpg", + "meta/instanceID": "uuid:5b2cc313-fc09-437e-8149-fcd32f695d41", + "_uuid": "5b2cc313-fc09-437e-8149-fcd32f695d41", + "_submission_time": "2013-02-14T15:37:21", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "1", + "_media_count": "0", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + "available_transportation_types_to_referral_facility/bicycle": "True", + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", + "_uuid": "f3d8dc65-91a6-4d0f-9e97-802128083390", + "_submission_time": "2013-02-14T15:37:22", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + self.ambulance_key: "weekly", + "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", + "_uuid": "9c6f3468-cfda-46e8-84c1-75458e72805d", + "_submission_time": "2013-02-14T15:37:23", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/taxi": "True", + "available_transportation_types_to_referral_facility/other": "True", + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_uuid": "9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_submission_time": "2013-02-14T15:37:24", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, ] dd = DataDictionary.objects.get(pk=self.xform.pk) - additional_headers = dd._additional_headers() + [ - '_id', '_date_modified'] + additional_headers = dd._additional_headers() + ["_id", "_date_modified"] for row, expected_dict in zip(actual_csv, data): test_dict = {} d = dict(zip(headers, row)) @@ -447,7 +503,7 @@ def _check_csv_export_second_pass(self): test_dict[k] = v this_list = [] for k, v in expected_dict.items(): - if k in ['image1', 'meta/instanceID'] or k.startswith("_"): + if k in ["image1", "meta/instanceID"] or k.startswith("_"): this_list.append((k, v)) else: this_list.append(("transport/" + k, v)) @@ -461,20 +517,25 @@ def test_xls_export_content(self): def _check_xls_export(self): xls_export_url = reverse( - 'xls_export', kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + "xls_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(xls_export_url) - expected_xls = open_workbook(os.path.join( - self.this_directory, "fixtures", "transportation", - "transportation_export.xls")) + expected_xls = open_workbook( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_export.xls", + ) + ) content = get_response_content(response, decode=False) actual_xls = open_workbook(file_contents=content) actual_sheet = actual_xls.sheet_by_index(0) expected_sheet = expected_xls.sheet_by_index(0) # check headers - self.assertEqual(actual_sheet.row_values(0), - expected_sheet.row_values(0)) + self.assertEqual(actual_sheet.row_values(0), expected_sheet.row_values(0)) # check cell data self.assertEqual(actual_sheet.ncols, expected_sheet.ncols) @@ -494,10 +555,9 @@ def _check_delete(self): self.assertEquals(self.user.xforms.count(), 0) def test_405_submission(self): - url = reverse('submissions') + url = reverse("submissions") response = self.client.get(url) - self.assertContains( - response, 'Method "GET" not allowed', status_code=405) + self.assertContains(response, 'Method "GET" not allowed', status_code=405) def test_publish_bad_xls_with_unicode_in_error(self): """ @@ -507,24 +567,24 @@ def test_publish_bad_xls_with_unicode_in_error(self): """ self._create_user_and_login() path = os.path.join( - self.this_directory, 'fixtures', - 'form_with_unicode_in_relevant_column.xlsx') - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - response = self.client.post('/%s/' % self.user.username, post_data) + self.this_directory, "fixtures", "form_with_unicode_in_relevant_column.xlsx" + ) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + response = self.client.post("/%s/" % self.user.username, post_data) self.assertEqual(response.status_code, 200) def test_metadata_file_hash(self): self._publish_transportation_form() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + uf = UploadedFile(file=open(src, "rb"), content_type="image/png") count = MetaData.objects.count() MetaData.media_upload(self.xform, uf) # assert successful insert of new metadata record self.assertEqual(MetaData.objects.count(), count + 1) - md = MetaData.objects.get(object_id=self.xform.id, - data_value='screenshot.png') + md = MetaData.objects.get(object_id=self.xform.id, data_value="screenshot.png") # assert checksum string has been generated, hash length > 1 self.assertTrue(len(md.hash) > 16) @@ -534,13 +594,16 @@ def test_uuid_injection_in_cascading_select(self): """ pre_count = XForm.objects.count() xls_path = os.path.join( - self.this_directory, "fixtures", "cascading_selects", - "new_cascading_select.xlsx") + self.this_directory, + "fixtures", + "cascading_selects", + "new_cascading_select.xlsx", + ) file_name, file_ext = os.path.splitext(os.path.split(xls_path)[1]) TestBase._publish_xls_file(self, xls_path) post_count = XForm.objects.count() self.assertEqual(post_count, pre_count + 1) - xform = XForm.objects.latest('date_created') + xform = XForm.objects.latest("date_created") # check that the uuid is within the main instance/ # the one without an id attribute @@ -548,18 +611,24 @@ def test_uuid_injection_in_cascading_select(self): # check for instance nodes that are direct children of the model node model_node = xml.getElementsByTagName("model")[0] - instance_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and - not node.hasAttribute("id")] + instance_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") + ] self.assertEqual(len(instance_nodes), 1) instance_node = instance_nodes[0] # get the first element whose id attribute is equal to our form's # id_string - form_nodes = [node for node in instance_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("id") == xform.id_string] + form_nodes = [ + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("id") == xform.id_string + ] form_node = form_nodes[0] # find the formhub node that has a uuid child node @@ -569,26 +638,31 @@ def test_uuid_injection_in_cascading_select(self): self.assertEqual(len(uuid_nodes), 1) # check for the calculate bind - calculate_bind_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName == "bind" and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + calculate_bind_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName == "bind" + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(calculate_bind_nodes), 1) calculate_bind_node = calculate_bind_nodes[0] self.assertEqual( - calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid) + calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid + ) def test_csv_publishing(self): - csv_text = '\n'.join([ - 'survey,,', ',type,name,label', - ',text,whatsyourname,"What is your name?"', 'choices,,']) - url = reverse('user_profile', - kwargs={'username': self.user.username}) + csv_text = "\n".join( + [ + "survey,,", + ",type,name,label", + ',text,whatsyourname,"What is your name?"', + "choices,,", + ] + ) + url = reverse("user_profile", kwargs={"username": self.user.username}) num_xforms = XForm.objects.count() - params = { - 'text_xls_form': csv_text - } + params = {"text_xls_form": csv_text} self.response = self.client.post(url, params) self.assertEqual(XForm.objects.count(), num_xforms + 1) @@ -596,10 +670,9 @@ def test_truncate_xform_title_to_255(self): self._publish_transportation_form() title = "a" * (XFORM_TITLE_LENGTH + 1) groups = re.match( - r"(.+)([^<]+)(.*)", - self.xform.xml, re.DOTALL).groups() - self.xform.xml = "{0}{1}{2}".format( - groups[0], title, groups[2]) + r"(.+)([^<]+)(.*)", self.xform.xml, re.DOTALL + ).groups() + self.xform.xml = "{0}{1}{2}".format(groups[0], title, groups[2]) self.xform.title = title self.xform.save() self.assertEqual(self.xform.title, "a" * XFORM_TITLE_LENGTH) diff --git a/onadata/apps/main/tests/test_user_settings.py b/onadata/apps/main/tests/test_user_settings.py index 0869a1e8bd..9d91863090 100644 --- a/onadata/apps/main/tests/test_user_settings.py +++ b/onadata/apps/main/tests/test_user_settings.py @@ -1,5 +1,5 @@ from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.main.models import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -7,18 +7,18 @@ class TestUserSettings(TestBase): - def setUp(self): TestBase.setUp(self) self.settings_url = reverse( - profile_settings, kwargs={'username': self.user.username}) + profile_settings, kwargs={"username": self.user.username} + ) def test_render_user_settings(self): response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 200) def test_access_user_settings_non_owner(self): - self._create_user_and_login('alice', 'alice') + self._create_user_and_login("alice", "alice") response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 404) @@ -31,14 +31,14 @@ def test_show_existing_profile_data(self): def test_update_user_settings(self): post_data = { - 'name': 'Bobby', - 'organization': 'Bob Inc', - 'city': 'Bobville', - 'country': 'BB', - 'twitter': 'bobo', - 'home_page': 'bob.com', - 'require_auth': True, - 'email': 'bob@bob.com' + "name": "Bobby", + "organization": "Bob Inc", + "city": "Bobville", + "country": "BB", + "twitter": "bobo", + "home_page": "bob.com", + "require_auth": True, + "email": "bob@bob.com", } response = self.client.post(self.settings_url, post_data) self.assertEqual(response.status_code, 302) @@ -47,7 +47,7 @@ def test_update_user_settings(self): try: self.assertEqual(self.user.profile.__dict__[key], value) except KeyError as e: - if key == 'email': + if key == "email": self.assertEqual(self.user.__dict__[key], value) else: raise e diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 95be00081c..24101e8980 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -3,7 +3,8 @@ RestService model """ import importlib -from future.utils import python_2_unicode_compatible + +from six import python_2_unicode_compatible from django.conf import settings from django.db import models @@ -23,30 +24,32 @@ class RestService(models.Model): """ class Meta: - app_label = 'restservice' - unique_together = ('service_url', 'xform', 'name') + app_label = "restservice" + unique_together = ("service_url", "xform", "name") service_url = models.URLField(ugettext_lazy("Service URL")) xform = models.ForeignKey(XForm, on_delete=models.CASCADE) name = models.CharField(max_length=50, choices=SERVICE_CHOICES) - date_created = models.DateTimeField( - auto_now_add=True, null=True, blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) - active = models.BooleanField(ugettext_lazy("Active"), default=True, - blank=False, null=False) - inactive_reason = models.TextField(ugettext_lazy("Inactive reason"), - blank=True, default="") + active = models.BooleanField( + ugettext_lazy("Active"), default=True, blank=False, null=False + ) + inactive_reason = models.TextField( + ugettext_lazy("Inactive reason"), blank=True, default="" + ) def __str__(self): - return u"%s:%s - %s" % (self.xform, self.long_name, self.service_url) + return "%s:%s - %s" % (self.xform, self.long_name, self.service_url) def get_service_definition(self): """ Returns ServiceDefinition class """ - services_to_modules = getattr(settings, 'REST_SERVICES_TO_MODULES', {}) + services_to_modules = getattr(settings, "REST_SERVICES_TO_MODULES", {}) module_name = services_to_modules.get( - self.name, 'onadata.apps.restservice.services.%s' % self.name) + self.name, "onadata.apps.restservice.services.%s" % self.name + ) module = importlib.import_module(module_name) return module.ServiceDefinition @@ -67,12 +70,11 @@ def delete_metadata(sender, instance, **kwargs): # pylint: disable=W0613 """ if instance.name in [TEXTIT, GOOGLE_SHEET]: MetaData.objects.filter( # pylint: disable=no-member - object_id=instance.xform.id, - data_type=instance.name).delete() + object_id=instance.xform.id, data_type=instance.name + ).delete() -post_delete.connect( - delete_metadata, sender=RestService, dispatch_uid='delete_metadata') +post_delete.connect(delete_metadata, sender=RestService, dispatch_uid="delete_metadata") # pylint: disable=W0613 @@ -80,19 +82,19 @@ def propagate_merged_datasets(sender, instance, **kwargs): """ Propagate the service to the individual forms of a merged dataset. """ - created = kwargs.get('created') + created = kwargs.get("created") if created and instance.xform.is_merged_dataset: for xform in instance.xform.mergedxform.xforms.all(): RestService.objects.create( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) post_save.connect( propagate_merged_datasets, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) # pylint: disable=W0613 @@ -104,9 +106,8 @@ def delete_merged_datasets_service(sender, instance, **kwargs): for xform in instance.xform.mergedxform.xforms.all(): try: service = RestService.objects.get( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) except RestService.DoesNotExist: pass else: @@ -116,4 +117,5 @@ def delete_merged_datasets_service(sender, instance, **kwargs): post_delete.connect( delete_merged_datasets_service, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) diff --git a/onadata/apps/restservice/services/textit.py b/onadata/apps/restservice/services/textit.py index 3147e5075a..427372c0a0 100644 --- a/onadata/apps/restservice/services/textit.py +++ b/onadata/apps/restservice/services/textit.py @@ -1,6 +1,6 @@ import json import requests -from future.utils import iteritems +from six import iteritems from six import string_types from onadata.apps.main.models import MetaData @@ -11,7 +11,7 @@ class ServiceDefinition(RestServiceInterface): id = TEXTIT - verbose_name = u'TextIt POST' + verbose_name = "TextIt POST" def send(self, url, submission_instance): """ @@ -29,10 +29,12 @@ def send(self, url, submission_instance): post_data = { "extra": extra_data, "flow": flow, - "contacts": contacts.split(',') + "contacts": contacts.split(","), + } + headers = { + "Content-Type": "application/json", + "Authorization": "Token {}".format(token), } - headers = {"Content-Type": "application/json", - "Authorization": "Token {}".format(token)} requests.post(url, headers=headers, data=json.dumps(post_data)) @@ -48,14 +50,12 @@ def clean_keys_of_slashes(self, record): if not isinstance(value, string_types): record[key] = str(value) - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply # same - if value and isinstance(value, list)\ - and isinstance(value[0], dict): + if value and isinstance(value, list) and isinstance(value[0], dict): for v in value: self.clean_keys_of_slashes(v) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index f3c606e67a..0091d5f3e4 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -11,19 +11,21 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ from floip import FloipSurvey from kombu.exceptions import OperationalError from pyxform.builder import create_survey_element_from_dict from pyxform.utils import has_external_choices from pyxform.xls2json import parse_file_to_json +from six import python_2_unicode_compatible -from onadata.apps.logger.models.xform import (XForm, check_version_set, - check_xform_uuid) +from onadata.apps.logger.models.xform import XForm, check_version_set, check_xform_uuid from onadata.apps.logger.xform_instance_parser import XLSFormError -from onadata.libs.utils.cache_tools import (PROJ_BASE_FORMS_CACHE, - PROJ_FORMS_CACHE, safe_delete) +from onadata.libs.utils.cache_tools import ( + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + safe_delete, +) from onadata.libs.utils.model_tools import get_columns_with_hxl, set_uuid @@ -32,8 +34,10 @@ def is_newline_error(e): Return True is e is a new line error based on the error text. Otherwise return False. """ - newline_error = u'new-line character seen in unquoted field - do you need'\ - u' to open the file in universal-newline mode?' + newline_error = ( + "new-line character seen in unquoted field - do you need" + " to open the file in universal-newline mode?" + ) return newline_error == text(e) @@ -42,11 +46,11 @@ def process_xlsform(xls, default_name): Process XLSForm file and return the survey dictionary for the XLSForm. """ # FLOW Results package is a JSON file. - if xls.name.endswith('json'): + if xls.name.endswith("json"): return FloipSurvey(xls).survey.to_json_dict() file_object = None - if xls.name.endswith('csv'): + if xls.name.endswith("csv"): # a csv file gets closed in pyxform, make a copy xls.seek(0) file_object = BytesIO() @@ -59,10 +63,10 @@ def process_xlsform(xls, default_name): except csv.Error as e: if is_newline_error(e): xls.seek(0) - file_object = StringIO( - u'\n'.join(xls.read().splitlines())) + file_object = StringIO("\n".join(xls.read().splitlines())) return parse_file_to_json( - xls.name, default_name=default_name, file_object=file_object) + xls.name, default_name=default_name, file_object=file_object + ) raise e @@ -80,12 +84,13 @@ def sheet_to_csv(xls_content, sheet_name): sheet = workbook.sheet_by_name(sheet_name) if not sheet or sheet.nrows < 2: - raise Exception(_(u"Sheet <'%(sheet_name)s'> has no data." % - {'sheet_name': sheet_name})) + raise Exception( + _("Sheet <'%(sheet_name)s'> has no data." % {"sheet_name": sheet_name}) + ) csv_file = BytesIO() - writer = csv.writer(csv_file, encoding='utf-8', quoting=csv.QUOTE_ALL) + writer = csv.writer(csv_file, encoding="utf-8", quoting=csv.QUOTE_ALL) mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] header = [v for v, m in zip(sheet.row_values(0), mask) if m] @@ -93,7 +98,7 @@ def sheet_to_csv(xls_content, sheet_name): name_column = None try: - name_column = header.index('name') + name_column = header.index("name") except ValueError: pass @@ -114,19 +119,15 @@ def sheet_to_csv(xls_content, sheet_name): for index, val in enumerate(sheet.row_values(row)): if sheet.cell_type(row, index) == xlrd.XL_CELL_NUMBER: try: - val = str( - float(val) if ( - float(val) > int(val)) else int(val)) + val = str(float(val) if (float(val) > int(val)) else int(val)) except ValueError: pass elif sheet.cell_type(row, index) == xlrd.XL_CELL_DATE: - val = xlrd.xldate_as_datetime( - val, workbook.datemode).isoformat() + val = xlrd.xldate_as_datetime(val, workbook.datemode).isoformat() row_values.append(val) writer.writerow([v for v, m in zip(row_values, mask) if m]) else: - writer.writerow( - [v for v, m in zip(sheet.row_values(row), mask) if m]) + writer.writerow([v for v, m in zip(sheet.row_values(row), mask) if m]) return csv_file @@ -137,11 +138,7 @@ def upload_to(instance, filename, username=None): """ if instance: username = instance.xform.user.username - return os.path.join( - username, - 'xls', - os.path.split(filename)[1] - ) + return os.path.join(username, "xls", os.path.split(filename)[1]) @python_2_unicode_compatible @@ -160,52 +157,56 @@ def __str__(self): return getattr(self, "id_string", "") def save(self, *args, **kwargs): - skip_xls_read = kwargs.get('skip_xls_read') + skip_xls_read = kwargs.get("skip_xls_read") if self.xls and not skip_xls_read: - default_name = None \ - if not self.pk else self.survey.xml_instance().tagName + default_name = None if not self.pk else self.survey.xml_instance().tagName survey_dict = process_xlsform(self.xls, default_name) if has_external_choices(survey_dict): self.has_external_choices = True survey = create_survey_element_from_dict(survey_dict) survey = check_version_set(survey) - if get_columns_with_hxl(survey.get('children')): + if get_columns_with_hxl(survey.get("children")): self.has_hxl_support = True # if form is being replaced, don't check for id_string uniqueness if self.pk is None: - new_id_string = self.get_unique_id_string( - survey.get('id_string')) - self._id_string_changed = \ - new_id_string != survey.get('id_string') - survey['id_string'] = new_id_string + new_id_string = self.get_unique_id_string(survey.get("id_string")) + self._id_string_changed = new_id_string != survey.get("id_string") + survey["id_string"] = new_id_string # For flow results packages use the user defined id/uuid - if self.xls.name.endswith('json'): - self.uuid = FloipSurvey(self.xls).descriptor.get('id') + if self.xls.name.endswith("json"): + self.uuid = FloipSurvey(self.xls).descriptor.get("id") if self.uuid: check_xform_uuid(self.uuid) - elif self.id_string != survey.get('id_string'): - raise XLSFormError(_( - (u"Your updated form's id_string '%(new_id)s' must match " - "the existing forms' id_string '%(old_id)s'." % { - 'new_id': survey.get('id_string'), - 'old_id': self.id_string}))) - elif default_name and default_name != survey.get('name'): - survey['name'] = default_name + elif self.id_string != survey.get("id_string"): + raise XLSFormError( + _( + ( + "Your updated form's id_string '%(new_id)s' must match " + "the existing forms' id_string '%(old_id)s'." + % { + "new_id": survey.get("id_string"), + "old_id": self.id_string, + } + ) + ) + ) + elif default_name and default_name != survey.get("name"): + survey["name"] = default_name else: - survey['id_string'] = self.id_string + survey["id_string"] = self.id_string self.json = survey.to_json() self.xml = survey.to_xml() - self.version = survey.get('version') + self.version = survey.get("version") self.last_updated_at = timezone.now() - self.title = survey.get('title') + self.title = survey.get("title") self._mark_start_time_boolean() set_uuid(self) self._set_uuid_in_xml() self._set_hash() - if 'skip_xls_read' in kwargs: - del kwargs['skip_xls_read'] + if "skip_xls_read" in kwargs: + del kwargs["skip_xls_read"] super(DataDictionary, self).save(*args, **kwargs) @@ -221,8 +222,8 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): """ if instance.project: # clear cache - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, instance.project.pk)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, instance.project.pk)) + safe_delete("{}{}".format(PROJ_FORMS_CACHE, instance.project.pk)) + safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, instance.project.pk)) # seems the super is not called, have to get xform from here xform = XForm.objects.get(pk=instance.pk) @@ -235,37 +236,45 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if instance.created_by and instance.user != instance.created_by: OwnerRole.add(instance.created_by, xform) - from onadata.libs.utils.project_utils import set_project_perms_to_xform_async # noqa + from onadata.libs.utils.project_utils import ( + set_project_perms_to_xform_async, + ) # noqa + try: - set_project_perms_to_xform_async.delay(xform.pk, - instance.project.pk) + set_project_perms_to_xform_async.delay(xform.pk, instance.project.pk) except OperationalError: - from onadata.libs.utils.project_utils import set_project_perms_to_xform # noqa + from onadata.libs.utils.project_utils import ( + set_project_perms_to_xform, + ) # noqa + set_project_perms_to_xform(xform, instance.project) - if hasattr(instance, 'has_external_choices') \ - and instance.has_external_choices: + if hasattr(instance, "has_external_choices") and instance.has_external_choices: instance.xls.seek(0) - f = sheet_to_csv(instance.xls.read(), 'external_choices') + f = sheet_to_csv(instance.xls.read(), "external_choices") f.seek(0, os.SEEK_END) size = f.tell() f.seek(0) from onadata.apps.main.models.meta_data import MetaData + data_file = InMemoryUploadedFile( file=f, - field_name='data_file', - name='itemsets.csv', - content_type='text/csv', + field_name="data_file", + name="itemsets.csv", + content_type="text/csv", size=size, - charset=None + charset=None, ) MetaData.media_upload(xform, data_file) -post_save.connect(set_object_permissions, sender=DataDictionary, - dispatch_uid='xform_object_permissions') +post_save.connect( + set_object_permissions, + sender=DataDictionary, + dispatch_uid="xform_object_permissions", +) # pylint: disable=unused-argument @@ -277,5 +286,6 @@ def save_project(sender, instance=None, created=False, **kwargs): instance.project.save() -pre_save.connect(save_project, sender=DataDictionary, - dispatch_uid='save_project_datadictionary') +pre_save.connect( + save_project, sender=DataDictionary, dispatch_uid="save_project_datadictionary" +) diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 01f8dfde41..815d58b34c 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -3,7 +3,8 @@ Export model. """ import os -from future.utils import python_2_unicode_compatible + +from six import python_2_unicode_compatible from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class @@ -15,7 +16,7 @@ from onadata.libs.utils.common_tags import OSM from onadata.libs.utils import async_status -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" # pylint: disable=unused-argument @@ -23,7 +24,7 @@ def export_delete_callback(sender, **kwargs): """ Delete export file when an export object is deleted. """ - export = kwargs['instance'] + export = kwargs["instance"] storage = get_storage_class()() if export.filepath and storage.exists(export.filepath): storage.delete(export.filepath) @@ -38,7 +39,7 @@ def get_export_options_query_kwargs(options): if field in options: field_value = options.get(field) - key = 'options__{}'.format(field) + key = "options__{}".format(field) options_kwargs[key] = field_value return options_kwargs @@ -49,8 +50,9 @@ class ExportTypeError(Exception): """ ExportTypeError exception class. """ + def __str__(self): - return _(u'Invalid export type specified') + return _("Invalid export type specified") @python_2_unicode_compatible @@ -58,8 +60,9 @@ class ExportConnectionError(Exception): """ ExportConnectionError exception class. """ + def __str__(self): - return _(u'Export server is down.') + return _("Export server is down.") @python_2_unicode_compatible @@ -67,40 +70,41 @@ class Export(models.Model): """ Class representing a data export from an XForm """ - XLS_EXPORT = 'xls' - CSV_EXPORT = 'csv' - KML_EXPORT = 'kml' - ZIP_EXPORT = 'zip' - CSV_ZIP_EXPORT = 'csv_zip' - SAV_ZIP_EXPORT = 'sav_zip' - SAV_EXPORT = 'sav' - EXTERNAL_EXPORT = 'external' + + XLS_EXPORT = "xls" + CSV_EXPORT = "csv" + KML_EXPORT = "kml" + ZIP_EXPORT = "zip" + CSV_ZIP_EXPORT = "csv_zip" + SAV_ZIP_EXPORT = "sav_zip" + SAV_EXPORT = "sav" + EXTERNAL_EXPORT = "external" OSM_EXPORT = OSM - GOOGLE_SHEETS_EXPORT = 'gsheets' + GOOGLE_SHEETS_EXPORT = "gsheets" EXPORT_MIMES = { - 'xls': 'vnd.ms-excel', - 'xlsx': 'vnd.openxmlformats', - 'csv': 'csv', - 'zip': 'zip', - 'csv_zip': 'zip', - 'sav_zip': 'zip', - 'sav': 'sav', - 'kml': 'vnd.google-earth.kml+xml', - OSM: OSM + "xls": "vnd.ms-excel", + "xlsx": "vnd.openxmlformats", + "csv": "csv", + "zip": "zip", + "csv_zip": "zip", + "sav_zip": "zip", + "sav": "sav", + "kml": "vnd.google-earth.kml+xml", + OSM: OSM, } EXPORT_TYPES = [ - (XLS_EXPORT, 'Excel'), - (CSV_EXPORT, 'CSV'), - (ZIP_EXPORT, 'ZIP'), - (KML_EXPORT, 'kml'), - (CSV_ZIP_EXPORT, 'CSV ZIP'), - (SAV_ZIP_EXPORT, 'SAV ZIP'), - (SAV_EXPORT, 'SAV'), - (EXTERNAL_EXPORT, 'Excel'), + (XLS_EXPORT, "Excel"), + (CSV_EXPORT, "CSV"), + (ZIP_EXPORT, "ZIP"), + (KML_EXPORT, "kml"), + (CSV_ZIP_EXPORT, "CSV ZIP"), + (SAV_ZIP_EXPORT, "SAV ZIP"), + (SAV_EXPORT, "SAV"), + (EXTERNAL_EXPORT, "Excel"), (OSM, OSM), - (GOOGLE_SHEETS_EXPORT, 'Google Sheets'), + (GOOGLE_SHEETS_EXPORT, "Google Sheets"), ] EXPORT_OPTION_FIELDS = [ @@ -131,7 +135,7 @@ class Export(models.Model): MAX_EXPORTS = 10 # Required fields - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) export_type = models.CharField( max_length=10, choices=EXPORT_TYPES, default=XLS_EXPORT ) @@ -159,14 +163,15 @@ class Meta: unique_together = (("xform", "filename"),) def __str__(self): - return u'%s - %s (%s)' % (self.export_type, self.xform, self.filename) + return "%s - %s (%s)" % (self.export_type, self.xform, self.filename) def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.pk and self.xform: # if new, check if we've hit our limit for exports for this form, # if so, delete oldest num_existing_exports = Export.objects.filter( - xform=self.xform, export_type=self.export_type).count() + xform=self.xform, export_type=self.export_type + ).count() if num_existing_exports >= self.MAX_EXPORTS: Export._delete_oldest_export(self.xform, self.export_type) @@ -174,8 +179,7 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ # update time_of_last_submission with # xform.time_of_last_submission_update # pylint: disable=E1101 - self.time_of_last_submission = self.xform.\ - time_of_last_submission_update() + self.time_of_last_submission = self.xform.time_of_last_submission_update() if self.filename: self.internal_status = Export.SUCCESSFUL super(Export, self).save(*args, **kwargs) @@ -183,7 +187,8 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ @classmethod def _delete_oldest_export(cls, xform, export_type): oldest_export = Export.objects.filter( - xform=xform, export_type=export_type).order_by('created_on')[0] + xform=xform, export_type=export_type + ).order_by("created_on")[0] oldest_export.delete() @property @@ -227,9 +232,9 @@ def _update_filedir(self): if not self.filename: raise AssertionError() # pylint: disable=E1101 - self.filedir = os.path.join(self.xform.user.username, - 'exports', self.xform.id_string, - self.export_type) + self.filedir = os.path.join( + self.xform.user.username, "exports", self.xform.id_string, self.export_type + ) @property def filepath(self): @@ -273,16 +278,22 @@ def exports_outdated(cls, xform, export_type, options=None): try: export_options = get_export_options_query_kwargs(options) latest_export = Export.objects.filter( - xform=xform, export_type=export_type, + xform=xform, + export_type=export_type, internal_status__in=[Export.SUCCESSFUL, Export.PENDING], - **export_options).latest('created_on') + **export_options + ).latest("created_on") except cls.DoesNotExist: return True else: - if latest_export.time_of_last_submission is not None \ - and xform.time_of_last_submission_update() is not None: - return latest_export.time_of_last_submission <\ - xform.time_of_last_submission_update() + if ( + latest_export.time_of_last_submission is not None + and xform.time_of_last_submission_update() is not None + ): + return ( + latest_export.time_of_last_submission + < xform.time_of_last_submission_update() + ) # return true if we can't determine the status, to force # auto-generation @@ -293,8 +304,7 @@ def is_filename_unique(cls, xform, filename): """ Return True if the filename is unique. """ - return Export.objects.filter( - xform=xform, filename=filename).count() == 0 + return Export.objects.filter(xform=xform, filename=filename).count() == 0 post_delete.connect(export_delete_callback, sender=Export) diff --git a/onadata/apps/viewer/parsed_instance_tools.py b/onadata/apps/viewer/parsed_instance_tools.py index 9f9c179f7e..d166b6ed11 100644 --- a/onadata/apps/viewer/parsed_instance_tools.py +++ b/onadata/apps/viewer/parsed_instance_tools.py @@ -2,37 +2,35 @@ import six import datetime from builtins import str as text -from future.utils import iteritems from typing import Any, Tuple from onadata.libs.utils.common_tags import MONGO_STRFTIME, DATE_FORMAT -KNOWN_DATES = ['_submission_time'] +KNOWN_DATES = ["_submission_time"] NONE_JSON_FIELDS = { - '_submission_time': 'date_created', - '_date_modified': 'date_modified', - '_id': 'id', - '_version': 'version', - '_last_edited': 'last_edited' + "_submission_time": "date_created", + "_date_modified": "date_modified", + "_id": "id", + "_version": "version", + "_last_edited": "last_edited", } -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): if known_integers is None: known_integers = [] if known_dates is None: known_dates = [] if known_decimals is None: known_decimals = [] - _json_str = u"json->>%s" + _json_str = "json->>%s" if key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif key in known_decimals: - _json_str = u"CAST(json->>%s AS DECIMAL)" + _json_str = "CAST(json->>%s AS DECIMAL)" return _json_str @@ -41,33 +39,25 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): # using a dictionary here just incase we will need to filter using # other table columns where, where_params = [], [] - OPERANDS = { - '$gt': '>', - '$gte': '>=', - '$lt': '<', - '$lte': '<=', - '$i': '~*' - } - for (field_key, field_value) in iteritems(query): + OPERANDS = {"$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<=", "$i": "~*"} + for (field_key, field_value) in six.iteritems(query): if isinstance(field_value, dict): if field_key in NONE_JSON_FIELDS: json_str = NONE_JSON_FIELDS.get(field_key) else: json_str = _json_sql_str( - field_key, known_integers, KNOWN_DATES, known_decimals) - for (key, value) in iteritems(field_value): + field_key, known_integers, KNOWN_DATES, known_decimals + ) + for (key, value) in six.iteritems(field_value): _v = None if key in OPERANDS: - where.append( - u' '.join([json_str, OPERANDS.get(key), u'%s']) - ) + where.append(" ".join([json_str, OPERANDS.get(key), "%s"])) _v = value if field_key in KNOWN_DATES: raw_date = value for date_format in (MONGO_STRFTIME, DATE_FORMAT): try: - _v = datetime.datetime.strptime(raw_date[:19], - date_format) + _v = datetime.datetime.strptime(raw_date[:19], date_format) except ValueError: pass if field_key in NONE_JSON_FIELDS: @@ -81,7 +71,7 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): elif field_value is None: where.append(f"json->>'{field_key}' IS NULL") else: - where.append(u"json->>%s = %s") + where.append("json->>%s = %s") where_params.extend((field_key, text(field_value))) return where + or_where, where_params + or_params @@ -102,55 +92,54 @@ def _merge_duplicate_keys(pairs: Tuple[str, Any]): return ret -def get_where_clause(query, form_integer_fields=None, - form_decimal_fields=None): +def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): if form_integer_fields is None: form_integer_fields = [] if form_decimal_fields is None: form_decimal_fields = [] - known_integers = ['_id'] + form_integer_fields + known_integers = ["_id"] + form_integer_fields known_decimals = form_decimal_fields where = [] where_params = [] try: if query and isinstance(query, (dict, six.string_types)): - query = query if isinstance(query, dict) else json.loads( - query, object_pairs_hook=_merge_duplicate_keys) + query = ( + query + if isinstance(query, dict) + else json.loads(query, object_pairs_hook=_merge_duplicate_keys) + ) or_where = [] or_params = [] if isinstance(query, list): query = query[0] - if isinstance(query, dict) and '$or' in list(query): - or_dict = query.pop('$or') + if isinstance(query, dict) and "$or" in list(query): + or_dict = query.pop("$or") for or_query in or_dict: for k, v in or_query.items(): if v is None: - or_where.extend( - [u"json->>'{}' IS NULL".format(k)]) + or_where.extend(["json->>'{}' IS NULL".format(k)]) elif isinstance(v, list): for value in v: or_where.extend(["json->>%s = %s"]) or_params.extend([k, value]) else: - or_where.extend( - [u"json->>%s = %s"]) + or_where.extend(["json->>%s = %s"]) or_params.extend([k, v]) - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] - where, where_params = _parse_where(query, known_integers, - known_decimals, or_where, - or_params) + where, where_params = _parse_where( + query, known_integers, known_decimals, or_where, or_params + ) except (ValueError, AttributeError) as e: - if query and isinstance(query, six.string_types) and \ - query.startswith('{'): + if query and isinstance(query, six.string_types) and query.startswith("{"): raise e # cast query param to text - where = [u"json::text ~* cast(%s as text)"] + where = ["json::text ~* cast(%s as text)"] where_params = [query] return where, where_params diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index 54a846e38c..1126caa81d 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -5,7 +5,7 @@ import sys from datetime import timedelta -from future.utils import iteritems +from six import iteritems from django.conf import settings from django.shortcuts import get_object_or_404 @@ -17,21 +17,23 @@ from onadata.apps.viewer.models.export import Export, ExportTypeError from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.common_tools import get_boolean_value, report_exception -from onadata.libs.utils.export_tools import (generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export) +from onadata.libs.utils.export_tools import ( + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, +) from onadata.celery import app -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" def _get_export_object(export_id): try: return Export.objects.get(id=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): + if getattr(settings, "SLAVE_DATABASES", []): from multidb.pinning import use_master with use_master: @@ -41,11 +43,7 @@ def _get_export_object(export_id): def _get_export_details(username, id_string, export_id): - details = { - 'export_id': export_id, - 'username': username, - 'id_string': id_string - } + details = {"export_id": export_id, "username": username, "id_string": id_string} return details @@ -65,24 +63,27 @@ def _create_export(xform, export_type, options): for (key, value) in iteritems(options) if key in Export.EXPORT_OPTION_FIELDS } - if query and 'query' not in export_options: - export_options['query'] = query + if query and "query" not in export_options: + export_options["query"] = query return Export.objects.create( - xform=xform, export_type=export_type, options=export_options) + xform=xform, export_type=export_type, options=export_options + ) export = _create_export(xform, export_type, options) result = None export_id = export.id - options.update({ - 'username': username, - 'id_string': id_string, - 'export_id': export_id, - 'query': query, - 'force_xlsx': force_xlsx - }) + options.update( + { + "username": username, + "id_string": id_string, + "export_id": export_id, + "query": query, + "force_xlsx": force_xlsx, + } + ) export_types = { Export.XLS_EXPORT: create_xls_export, @@ -93,7 +94,7 @@ def _create_export(xform, export_type, options): Export.ZIP_EXPORT: create_zip_export, Export.KML_EXPORT: create_kml_export, Export.OSM_EXPORT: create_osm_export, - Export.EXTERNAL_EXPORT: create_external_export + Export.EXTERNAL_EXPORT: create_external_export, } # start async export @@ -113,7 +114,7 @@ def _create_export(xform, export_type, options): # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save - if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False): + if getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False): export = get_object_or_404(Export, id=export.id) export.task_id = result.task_id export.save() @@ -129,7 +130,7 @@ def create_xls_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) - options["extension"] = 'xlsx' if force_xlsx else 'xls' + options["extension"] = "xlsx" if force_xlsx else "xls" try: export = _get_export_object(export_id) @@ -140,8 +141,9 @@ def create_xls_export(username, id_string, export_id, **options): # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: - gen_export = generate_export(Export.XLS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.XLS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -150,8 +152,10 @@ def create_xls_export(username, id_string, export_id, **options): report_exception( "XLS Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise @@ -171,8 +175,9 @@ def create_csv_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_EXPORT, export.xform, export_id, options + ) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records @@ -187,8 +192,10 @@ def create_csv_export(username, id_string, export_id, **options): report_exception( "CSV Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -212,7 +219,8 @@ def create_kml_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -220,8 +228,10 @@ def create_kml_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "KML Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -245,7 +255,8 @@ def create_osm_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -254,8 +265,10 @@ def create_osm_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -274,7 +287,8 @@ def create_zip_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -283,13 +297,17 @@ def create_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + ) raise else: if not settings.TESTING_MODE: delete_export.apply_async( - (), {'export_id': gen_export.id}, - countdown=settings.ZIP_EXPORT_COUNTDOWN) + (), + {"export_id": gen_export.id}, + countdown=settings.ZIP_EXPORT_COUNTDOWN, + ) return gen_export.id @@ -303,8 +321,9 @@ def create_csv_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -313,8 +332,10 @@ def create_csv_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "CSV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -330,8 +351,9 @@ def create_sav_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.SAV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() @@ -339,8 +361,10 @@ def create_sav_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -361,7 +385,8 @@ def create_external_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -369,8 +394,10 @@ def create_external_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "External Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -387,8 +414,9 @@ def create_google_sheet_export(username, id_string, export_id, **options): export = _get_export_object(export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -396,8 +424,10 @@ def create_google_sheet_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Google Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -427,7 +457,8 @@ def mark_expired_pending_exports_as_failed(): # pylint: disable=invalid-name task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.PENDING, created_on__lt=time_threshold) + internal_status=Export.PENDING, created_on__lt=time_threshold + ) exports.update(internal_status=Export.FAILED) @@ -439,5 +470,6 @@ def delete_expired_failed_exports(): task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.FAILED, created_on__lt=time_threshold) + internal_status=Export.FAILED, created_on__lt=time_threshold + ) exports.delete() diff --git a/onadata/apps/viewer/tests/test_kml_export.py b/onadata/apps/viewer/tests/test_kml_export.py index 007c0d4b74..93c30c0329 100644 --- a/onadata/apps/viewer/tests/test_kml_export.py +++ b/onadata/apps/viewer/tests/test_kml_export.py @@ -1,7 +1,7 @@ import os from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.logger.models.instance import Instance from onadata.apps.main.tests.test_base import TestBase @@ -9,43 +9,46 @@ class TestKMLExport(TestBase): - def _publish_survey(self): self.this_directory = os.path.dirname(__file__) xls_path = self._fixture_path("gps", "gps.xls") TestBase._publish_xls_file(self, xls_path) def test_kml_export(self): - id_string = 'gps' + id_string = "gps" self._publish_survey() self._make_submissions_gps() - self.fixtures = os.path.join( - self.this_directory, 'fixtures', 'kml_export') + self.fixtures = os.path.join(self.this_directory, "fixtures", "kml_export") url = reverse( - kml_export, - kwargs={'username': self.user.username, 'id_string': id_string}) + kml_export, kwargs={"username": self.user.username, "id_string": id_string} + ) response = self.client.get(url) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=id_string, - geom__isnull=False - ).order_by('id') + xform__user=self.user, xform__id_string=id_string, geom__isnull=False + ).order_by("id") self.assertEqual(instances.count(), 2) # create a tuple of replacement data per instance - replacement_data = [["{:,}".format(x) for x in [ - i.pk, i.point.x, i.point.y]] for i in instances] + replacement_data = [ + ["{:,}".format(x) for x in [i.pk, i.point.x, i.point.y]] for i in instances + ] # assuming 2 instances, flatten and assign to template names - replacement_dict = dict(zip(['pk1', 'x1', 'y1', 'pk2', 'x2', 'y2'], - [i for s in replacement_data for i in s])) - - with open(os.path.join(self.fixtures, 'export.kml')) as f: + replacement_dict = dict( + zip( + ["pk1", "x1", "y1", "pk2", "x2", "y2"], + [i for s in replacement_data for i in s], + ) + ) + + with open(os.path.join(self.fixtures, "export.kml")) as f: expected_content = f.read() for (template_name, template_data) in iteritems(replacement_dict): expected_content = expected_content.replace( - '{{%s}}' % template_name, template_data) + "{{%s}}" % template_name, template_data + ) self.assertMultiLineEqual( - expected_content.strip(), - response.content.decode('utf-8').strip()) + expected_content.strip(), response.content.decode("utf-8").strip() + ) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 297f7de322..fe6711a66b 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -13,30 +13,28 @@ from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator -from future.utils import iteritems +from fsix import iteritems from rest_framework import negotiation -from rest_framework.renderers import (BaseRenderer, JSONRenderer, - StaticHTMLRenderer, TemplateHTMLRenderer) +from rest_framework.renderers import ( + BaseRenderer, + JSONRenderer, + StaticHTMLRenderer, + TemplateHTMLRenderer, +) from rest_framework.utils.encoders import JSONEncoder from rest_framework_xml.renderers import XMLRenderer from onadata.libs.utils.osm import get_combined_osm IGNORE_FIELDS = [ - 'formhub/uuid', - 'meta/contactID', - 'meta/deprecatedID', - 'meta/instanceID', - 'meta/sessionID', + "formhub/uuid", + "meta/contactID", + "meta/deprecatedID", + "meta/instanceID", + "meta/sessionID", ] -FORMLIST_MANDATORY_FIELDS = [ - 'formID', - 'name', - 'version', - 'hash', - 'downloadUrl' -] +FORMLIST_MANDATORY_FIELDS = ["formID", "name", "version", "hash", "downloadUrl"] def pairing(val1, val2): @@ -52,16 +50,19 @@ def floip_rows_list(data): """ Yields a row of FLOIP results data from dict data. """ - _submission_time = pytz.timezone('UTC').localize( - parse_datetime(data['_submission_time'])).isoformat() + _submission_time = ( + pytz.timezone("UTC") + .localize(parse_datetime(data["_submission_time"])) + .isoformat() + ) for i, key in enumerate(data, 1): - if not (key.startswith('_') or key in IGNORE_FIELDS): - instance_id = data['_id'] + if not (key.startswith("_") or key in IGNORE_FIELDS): + instance_id = data["_id"] yield [ _submission_time, # Timestamp int(pairing(instance_id, i)), # Row ID - data.get('meta/contactID', data.get('_submitted_by')), - data.get('meta/sessionID') or data.get('_uuid') or instance_id, + data.get("meta/contactID", data.get("_submitted_by")), + data.get("meta/sessionID") or data.get("_uuid") or instance_id, key, # Question ID data[key], # Response None, # Response Metadata @@ -94,13 +95,14 @@ class XLSRenderer(BaseRenderer): # pylint: disable=R0903 XLSRenderer - renders .xls spreadsheet documents with application/vnd.openxmlformats. """ - media_type = 'application/vnd.openxmlformats' - format = 'xls' + + media_type = "application/vnd.openxmlformats" + format = "xls" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data @@ -109,29 +111,32 @@ class XLSXRenderer(XLSRenderer): # pylint: disable=too-few-public-methods XLSRenderer - renders .xlsx spreadsheet documents with application/vnd.openxmlformats. """ - format = 'xlsx' + + format = "xlsx" class CSVRenderer(BaseRenderer): # pylint: disable=abstract-method, R0903 """ XLSRenderer - renders comma separated files (CSV) with text/csv. """ - media_type = 'text/csv' - format = 'csv' - charset = 'utf-8' + + media_type = "text/csv" + format = "csv" + charset = "utf-8" class CSVZIPRenderer(BaseRenderer): # pylint: disable=R0903 """ CSVZIPRenderer - renders a ZIP file that contains CSV files. """ - media_type = 'application/octet-stream' - format = 'csvzip' + + media_type = "application/octet-stream" + format = "csvzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -141,13 +146,14 @@ class SAVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SAVZIPRenderer - renders a ZIP file that contains SPSS SAV files. """ - media_type = 'application/octet-stream' - format = 'savzip' + + media_type = "application/octet-stream" + format = "savzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -157,9 +163,10 @@ class SurveyRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SurveyRenderer - renders XML data. """ - media_type = 'application/xml' - format = 'xml' - charset = 'utf-8' + + media_type = "application/xml" + format = "xml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data @@ -169,9 +176,10 @@ class KMLRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ KMLRenderer - renders KML XML data. """ - media_type = 'application/xml' - format = 'kml' - charset = 'utf-8' + + media_type = "application/xml" + format = "kml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data @@ -181,7 +189,8 @@ class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=R0903 """ GoogleSheetsRenderer = Google Sheets excel exports. """ - format = 'gsheets' + + format = "gsheets" class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): @@ -196,9 +205,7 @@ def filter_renderers(self, renderers, format): # pylint: disable=W0622 so that we only negotiation against those that accept that format. If there is no renderer available, we use MediaFileRenderer. """ - renderers = [ - renderer for renderer in renderers if renderer.format == format - ] + renderers = [renderer for renderer in renderers if renderer.format == format] if not renderers: renderers = [MediaFileRenderer()] @@ -209,14 +216,15 @@ class MediaFileRenderer(BaseRenderer): # pylint: disable=R0903 """ MediaFileRenderer - render binary media files. """ - media_type = '*/*' + + media_type = "*/*" format = None charset = None - render_style = 'binary' + render_style = "binary" def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data @@ -225,11 +233,11 @@ class XFormListRenderer(BaseRenderer): # pylint: disable=R0903 Renderer which serializes to XML. """ - media_type = 'text/xml' - format = 'xml' - charset = 'utf-8' - root_node = 'xforms' - element_node = 'xform' + media_type = "text/xml" + format = "xml" + charset = "utf-8" + root_node = "xforms" + element_node = "xform" xmlns = "http://openrosa.org/xforms/xformsList" def render(self, data, accepted_media_type=None, renderer_context=None): @@ -237,7 +245,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): Renders *obj* into serialized XML. """ if data is None: - return '' + return "" elif isinstance(data, six.string_types): return data @@ -245,7 +253,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement(self.root_node, {'xmlns': self.xmlns}) + xml.startElement(self.root_node, {"xmlns": self.xmlns}) self._to_xml(xml, data) @@ -281,6 +289,7 @@ class XFormManifestRenderer(XFormListRenderer): # pylint: disable=R0903 """ XFormManifestRenderer - render XFormManifest XML. """ + root_node = "manifest" element_node = "mediaFile" xmlns = "http://openrosa.org/xforms/xformsManifest" @@ -290,30 +299,32 @@ class TemplateXMLRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 """ TemplateXMLRenderer - Render XML template. """ - format = 'xml' - media_type = 'text/xml' + + format = "xml" + media_type = "text/xml" def render(self, data, accepted_media_type=None, renderer_context=None): renderer_context = renderer_context or {} - response = renderer_context['response'] + response = renderer_context["response"] if response and response.exception: - return XMLRenderer().render(data, accepted_media_type, - renderer_context) + return XMLRenderer().render(data, accepted_media_type, renderer_context) - return super(TemplateXMLRenderer, - self).render(data, accepted_media_type, renderer_context) + return super(TemplateXMLRenderer, self).render( + data, accepted_media_type, renderer_context + ) class InstanceXMLRenderer(XMLRenderer): """ InstanceXMLRenderer - Renders Instance XML """ - root_tag_name = 'submission-batch' - item_tag_name = 'submission-item' + + root_tag_name = "submission-batch" + item_tag_name = "submission-item" def _get_current_buffer_data(self): - if hasattr(self, 'stream'): + if hasattr(self, "stream"): ret = self.stream.getvalue() self.stream.truncate(0) self.stream.seek(0) @@ -327,10 +338,7 @@ def stream_data(self, data, serializer): xml = SimplerXMLGenerator(self.stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()} - ) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) yield self._get_current_buffer_data() @@ -376,9 +384,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()}) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) self._to_xml(xml, data) @@ -392,8 +398,8 @@ def _pop_xml_attributes(self, xml_dictionary: dict) -> Tuple[dict, dict]: attributes = {} for key, value in xml_dictionary.items(): - if key.startswith('@'): - attributes.update({key.replace('@', ''): value}) + if key.startswith("@"): + attributes.update({key.replace("@", ""): value}) del ret[key] return ret, attributes @@ -437,17 +443,19 @@ class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=R0903 """ StaticXMLRenderer - render static XML document. """ - format = 'xml' - media_type = 'text/xml' + + format = "xml" + media_type = "text/xml" class GeoJsonRenderer(BaseRenderer): # pylint: disable=R0903 """ GeoJsonRenderer - render .geojson data as json. """ - media_type = 'application/json' - format = 'geojson' - charset = 'utf-8' + + media_type = "application/json" + format = "geojson" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return json.dumps(data) @@ -457,15 +465,16 @@ class OSMRenderer(BaseRenderer): # pylint: disable=R0903 """ OSMRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + + media_type = "text/xml" + format = "osm" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): # Process error before making a list if isinstance(data, dict): - if 'detail' in data: - return u'' + data['detail'] + '' + if "detail" in data: + return "" + data["detail"] + "" # Combine/concatenate the list of osm files to one file def _list(list_or_item): @@ -483,43 +492,47 @@ class OSMExportRenderer(BaseRenderer): # pylint: disable=R0903, W0223 """ OSMExportRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + + media_type = "text/xml" + format = "osm" + charset = "utf-8" class DebugToolbarRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 """ DebugToolbarRenderer - render .debug as HTML. """ - media_type = 'text/html' - charset = 'utf-8' - format = 'debug' - template_name = 'debug.html' + + media_type = "text/html" + charset = "utf-8" + format = "debug" + template_name = "debug.html" def render(self, data, accepted_media_type=None, renderer_context=None): data = { - 'debug_data': - str( + "debug_data": str( JSONRenderer().render(data, renderer_context=renderer_context), - self.charset) + self.charset, + ) } return super(DebugToolbarRenderer, self).render( - data, accepted_media_type, renderer_context) + data, accepted_media_type, renderer_context + ) class ZipRenderer(BaseRenderer): # pylint: disable=R0903 """ ZipRenderer - render .zip files. """ - media_type = 'application/octet-stream' - format = 'zip' + + media_type = "application/octet-stream" + format = "zip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -529,6 +542,7 @@ class DecimalJSONRenderer(JSONRenderer): """ Extends the default json renderer to handle Decimal('NaN') values """ + encoder_class = DecimalEncoder @@ -536,19 +550,21 @@ class FLOIPRenderer(JSONRenderer): """ FLOIP Results data renderer. """ - media_type = 'application/vnd.org.flowinterop.results+json' - format = 'json' - charset = 'utf-8' + + media_type = "application/vnd.org.flowinterop.results+json" + format = "json" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): - request = renderer_context['request'] - response = renderer_context['response'] + request = renderer_context["request"] + response = renderer_context["response"] results = data - if request.method == 'GET' and response.status_code == 200: + if request.method == "GET" and response.status_code == 200: if isinstance(data, dict): results = [i for i in floip_rows_list(data)] else: results = [i for i in floip_list(data)] - return super(FLOIPRenderer, self).render(results, accepted_media_type, - renderer_context) + return super(FLOIPRenderer, self).render( + results, accepted_media_type, renderer_context + ) diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index d59ecdd959..9459353635 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -1,5 +1,5 @@ import json -from future.utils import listvalues +from six import itervalues from rest_framework import serializers @@ -13,7 +13,7 @@ def dict_key_for_value(_dict, value): """ This function is used to get key by value in a dictionary """ - return list(_dict)[listvalues(_dict).index(value)] + return list(_dict)[list(itervalues(_dict)).index(value)] def get_path(data, question_name, path_list): @@ -25,12 +25,12 @@ def get_path(data, question_name, path_list): :return: an xpath which is a string or None if name cannot be found :rtype: string or None """ - name = data.get('name') + name = data.get("name") if name == question_name: - return '/'.join(path_list) - elif data.get('children') is not None: - for node in data.get('children'): - path_list.append(node.get('name')) + return "/".join(path_list) + elif data.get("children") is not None: + for node in data.get("children"): + path_list.append(node.get("name")) path = get_path(node, question_name, path_list) if path is not None: return path @@ -40,26 +40,35 @@ def get_path(data, question_name, path_list): class AttachmentSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='attachment-detail', - lookup_field='pk') + url = serializers.HyperlinkedIdentityField( + view_name="attachment-detail", lookup_field="pk" + ) field_xpath = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() small_download_url = serializers.SerializerMethodField() medium_download_url = serializers.SerializerMethodField() - xform = serializers.ReadOnlyField(source='instance.xform.pk') - instance = serializers.PrimaryKeyRelatedField( - queryset=Instance.objects.all()) - filename = serializers.ReadOnlyField(source='media_file.name') + xform = serializers.ReadOnlyField(source="instance.xform.pk") + instance = serializers.PrimaryKeyRelatedField(queryset=Instance.objects.all()) + filename = serializers.ReadOnlyField(source="media_file.name") class Meta: - fields = ('url', 'filename', 'mimetype', 'field_xpath', 'id', 'xform', - 'instance', 'download_url', 'small_download_url', - 'medium_download_url') + fields = ( + "url", + "filename", + "mimetype", + "field_xpath", + "id", + "xform", + "instance", + "download_url", + "small_download_url", + "medium_download_url", + ) model = Attachment @check_obj def get_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") if obj: path = get_attachment_url(obj) @@ -67,18 +76,18 @@ def get_download_url(self, obj): return request.build_absolute_uri(path) if request else path def get_small_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'small') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "small") return request.build_absolute_uri(path) if request else path def get_medium_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'medium') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "medium") return request.build_absolute_uri(path) if request else path diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 5444c0b552..967dbc56bb 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -13,7 +13,7 @@ from django.db.utils import IntegrityError from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from rest_framework import serializers from rest_framework.reverse import reverse @@ -21,42 +21,45 @@ from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models import MetaData from onadata.libs.permissions import ROLES, ManagerRole -from onadata.libs.serializers.fields.instance_related_field import \ - InstanceRelatedField -from onadata.libs.serializers.fields.project_related_field import \ - ProjectRelatedField -from onadata.libs.serializers.fields.xform_related_field import \ - XFormRelatedField +from onadata.libs.serializers.fields.instance_related_field import InstanceRelatedField +from onadata.libs.serializers.fields.project_related_field import ProjectRelatedField +from onadata.libs.serializers.fields.xform_related_field import XFormRelatedField from onadata.libs.utils.common_tags import ( - XFORM_META_PERMS, SUBMISSION_REVIEW, IMPORTED_VIA_CSV_BY) + XFORM_META_PERMS, + SUBMISSION_REVIEW, + IMPORTED_VIA_CSV_BY, +) -UNIQUE_TOGETHER_ERROR = u"Object already exists" +UNIQUE_TOGETHER_ERROR = "Object already exists" -CSV_CONTENT_TYPE = 'text/csv' -MEDIA_TYPE = 'media' -DOC_TYPE = 'supporting_doc' +CSV_CONTENT_TYPE = "text/csv" +MEDIA_TYPE = "media" +DOC_TYPE = "supporting_doc" METADATA_TYPES = ( - ('data_license', _(u"Data License")), - ('enketo_preview_url', _(u"Enketo Preview URL")), - ('enketo_url', _(u"Enketo URL")), - ('form_license', _(u"Form License")), - ('mapbox_layer', _(u"Mapbox Layer")), - (MEDIA_TYPE, _(u"Media")), - ('public_link', _(u"Public Link")), - ('source', _(u"Source")), - (DOC_TYPE, _(u"Supporting Document")), - ('external_export', _(u"External Export")), - ('textit', _(u"TextIt")), - ('google_sheets', _(u"Google Sheet")), - ('xform_meta_perms', _("Xform meta permissions")), - ('submission_review', _("Submission Review")), - (IMPORTED_VIA_CSV_BY, _("Imported via CSV by"))) # yapf:disable - -DATAVIEW_TAG = 'dataview' -XFORM_TAG = 'xform' - -PROJECT_METADATA_TYPES = ((MEDIA_TYPE, _(u"Media")), - ('supporting_doc', _(u"Supporting Document"))) + ("data_license", _("Data License")), + ("enketo_preview_url", _("Enketo Preview URL")), + ("enketo_url", _("Enketo URL")), + ("form_license", _("Form License")), + ("mapbox_layer", _("Mapbox Layer")), + (MEDIA_TYPE, _("Media")), + ("public_link", _("Public Link")), + ("source", _("Source")), + (DOC_TYPE, _("Supporting Document")), + ("external_export", _("External Export")), + ("textit", _("TextIt")), + ("google_sheets", _("Google Sheet")), + ("xform_meta_perms", _("Xform meta permissions")), + ("submission_review", _("Submission Review")), + (IMPORTED_VIA_CSV_BY, _("Imported via CSV by")), +) # yapf:disable + +DATAVIEW_TAG = "dataview" +XFORM_TAG = "xform" + +PROJECT_METADATA_TYPES = ( + (MEDIA_TYPE, _("Media")), + ("supporting_doc", _("Supporting Document")), +) def get_linked_object(parts): @@ -73,12 +76,14 @@ def get_linked_object(parts): try: obj_pk = int(obj_pk) except ValueError: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid %(type)s id %(id)s." % - {'type': obj_type, - 'id': obj_pk}) - }) + raise serializers.ValidationError( + { + "data_value": _( + "Invalid %(type)s id %(id)s." + % {"type": obj_type, "id": obj_pk} + ) + } + ) else: model = DataView if obj_type == DATAVIEW_TAG else XForm @@ -89,17 +94,17 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): """ MetaData HyperlinkedModelSerializer """ + id = serializers.ReadOnlyField() # pylint: disable=C0103 xform = XFormRelatedField(queryset=XForm.objects.all(), required=False) - project = ProjectRelatedField( - queryset=Project.objects.all(), required=False) - instance = InstanceRelatedField( - queryset=Instance.objects.all(), required=False) + project = ProjectRelatedField(queryset=Project.objects.all(), required=False) + instance = InstanceRelatedField(queryset=Instance.objects.all(), required=False) data_value = serializers.CharField(max_length=255, required=True) data_type = serializers.ChoiceField(choices=METADATA_TYPES) data_file = serializers.FileField(required=False) data_file_type = serializers.CharField( - max_length=255, required=False, allow_blank=True) + max_length=255, required=False, allow_blank=True + ) media_url = serializers.SerializerMethodField() date_created = serializers.ReadOnlyField() @@ -108,102 +113,133 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = MetaData - fields = ('id', 'xform', 'project', 'instance', 'data_value', - 'data_type', 'data_file', 'data_file_type', 'media_url', - 'file_hash', 'url', 'date_created') + fields = ( + "id", + "xform", + "project", + "instance", + "data_value", + "data_type", + "data_file", + "data_file_type", + "media_url", + "file_hash", + "url", + "date_created", + ) def get_media_url(self, obj): """ Returns media URL for given metadata """ - if obj.data_type in [DOC_TYPE, MEDIA_TYPE] and\ - getattr(obj, "data_file") and getattr(obj.data_file, "url"): + if ( + obj.data_type in [DOC_TYPE, MEDIA_TYPE] + and getattr(obj, "data_file") + and getattr(obj.data_file, "url") + ): return obj.data_file.url elif obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: kwargs = { - 'kwargs': { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "kwargs": { + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, }, - 'request': self.context.get('request'), - 'format': 'csv' + "request": self.context.get("request"), + "format": "csv", } - return reverse('xform-media', **kwargs) + return reverse("xform-media", **kwargs) def validate(self, attrs): """ Validate url if we are adding a media uri instead of a media file """ - value = attrs.get('data_value') - data_type = attrs.get('data_type') - data_file = attrs.get('data_file') - - if not ('project' in attrs or 'xform' in attrs or 'instance' in attrs): - raise serializers.ValidationError({ - 'missing_field': - _(u"`xform` or `project` or `instance`" - "field is required.") - }) + value = attrs.get("data_value") + data_type = attrs.get("data_type") + data_file = attrs.get("data_file") + + if not ("project" in attrs or "xform" in attrs or "instance" in attrs): + raise serializers.ValidationError( + { + "missing_field": _( + "`xform` or `project` or `instance`" "field is required." + ) + } + ) if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type not in allowed_types: - raise serializers.ValidationError({ - 'data_file': - _('Unsupported media file type %s' % data_content_type)}) + raise serializers.ValidationError( + { + "data_file": _( + "Unsupported media file type %s" % data_content_type + ) + } + ) else: - attrs['data_file_type'] = data_content_type + attrs["data_file_type"] = data_content_type - if data_type == 'media' and data_file is None: + if data_type == "media" and data_file is None: try: URLValidator()(value) except ValidationError: parts = value.split() if len(parts) < 3: - raise serializers.ValidationError({ - 'data_value': - _(u"Expecting 'xform [xform id] [media name]' " - "or 'dataview [dataview id] [media name]' " - "or a valid URL.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "Expecting 'xform [xform id] [media name]' " + "or 'dataview [dataview id] [media name]' " + "or a valid URL." + ) + } + ) obj = get_linked_object(parts) if obj: xform = obj.xform if isinstance(obj, DataView) else obj - request = self.context['request'] + request = self.context["request"] user_has_role = ManagerRole.user_has_role - has_perm = user_has_role(request.user, xform) or \ - user_has_role(request.user, obj.project) + has_perm = user_has_role(request.user, xform) or user_has_role( + request.user, obj.project + ) if not has_perm: - raise serializers.ValidationError({ - 'data_value': - _(u"User has no permission to " - "the dataview.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "User has no permission to " "the dataview." + ) + } + ) else: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid url '%s'." % value) - }) + raise serializers.ValidationError( + {"data_value": _("Invalid url '%s'." % value)} + ) else: # check if we have a value for the filename. if not os.path.basename(urlparse(value).path): - raise serializers.ValidationError({ - 'data_value': - _(u"Cannot get filename from URL %s. URL should " - u"include the filename e.g " - u"http://example.com/data.csv" % value) - }) + raise serializers.ValidationError( + { + "data_value": _( + "Cannot get filename from URL %s. URL should " + "include the filename e.g " + "http://example.com/data.csv" % value + ) + } + ) if data_type == XFORM_META_PERMS: - perms = value.split('|') + perms = value.split("|") if len(perms) != 2 or not set(perms).issubset(set(ROLES)): raise serializers.ValidationError( - _(u"Format 'role'|'role' or Invalid role")) + _("Format 'role'|'role' or Invalid role") + ) return attrs @@ -215,34 +251,38 @@ def get_content_object(self, validated_data): """ if validated_data: - return (validated_data.get('xform') or - validated_data.get('project') or - validated_data.get('instance')) + return ( + validated_data.get("xform") + or validated_data.get("project") + or validated_data.get("instance") + ) def create(self, validated_data): - data_type = validated_data.get('data_type') - data_file = validated_data.get('data_file') - data_file_type = validated_data.get('data_file_type') + data_type = validated_data.get("data_type") + data_file = validated_data.get("data_file") + data_file_type = validated_data.get("data_file_type") content_object = self.get_content_object(validated_data) - data_value = data_file.name \ - if data_file else validated_data.get('data_value') + data_value = data_file.name if data_file else validated_data.get("data_value") # not exactly sure what changed in the requests.FILES for django 1.7 # csv files uploaded in windows do not have the text/csv content_type # this works around that - if data_type == MEDIA_TYPE and data_file \ - and data_file.name.lower().endswith('.csv') \ - and data_file_type != CSV_CONTENT_TYPE: + if ( + data_type == MEDIA_TYPE + and data_file + and data_file.name.lower().endswith(".csv") + and data_file_type != CSV_CONTENT_TYPE + ): data_file_type = CSV_CONTENT_TYPE content_type = ContentType.objects.get_for_model(content_object) try: if data_type == XFORM_META_PERMS: - metadata = \ - MetaData.xform_meta_permission(content_object, - data_value=data_value) + metadata = MetaData.xform_meta_permission( + content_object, data_value=data_value + ) update_role_by_meta_xform_perms(content_object) elif data_type == SUBMISSION_REVIEW: @@ -251,10 +291,12 @@ def create(self, validated_data): raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) else: metadata = MetaData.submission_review( - content_object, data_value=data_value) + content_object, data_value=data_value + ) elif data_type == IMPORTED_VIA_CSV_BY: metadata = MetaData.instance_csv_imported_by( - content_object, data_value=data_value) + content_object, data_value=data_value + ) else: metadata = MetaData.objects.create( content_type=content_type, @@ -262,15 +304,15 @@ def create(self, validated_data): data_value=data_value, data_file=data_file, data_file_type=data_file_type, - object_id=content_object.id) + object_id=content_object.id, + ) return metadata except IntegrityError: raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) def update(self, instance, validated_data): - instance = super(MetaDataSerializer, self).update( - instance, validated_data) + instance = super(MetaDataSerializer, self).update(instance, validated_data) if instance.data_type == XFORM_META_PERMS: update_role_by_meta_xform_perms(instance.content_object) diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index ce292cf95e..1eaeb47fd4 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -1,5 +1,5 @@ from builtins import bytes as b -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from django.contrib.auth.models import User from django.contrib.auth.tokens import PasswordResetTokenGenerator @@ -16,44 +16,55 @@ class CustomPasswordResetTokenGenerator(PasswordResetTokenGenerator): """Custom Password Token Generator Class.""" + def _make_hash_value(self, user, timestamp): # Include user email alongside user password to the generated token # as the user state object that might change after a password reset # to produce a token that invalidated. - login_timestamp = '' if user.last_login is None\ + login_timestamp = ( + "" + if user.last_login is None else user.last_login.replace(microsecond=0, tzinfo=None) - return str(user.pk) + user.password + user.email +\ - str(login_timestamp) + str(timestamp) + ) + return ( + str(user.pk) + + user.password + + user.email + + str(login_timestamp) + + str(timestamp) + ) default_token_generator = CustomPasswordResetTokenGenerator() -def get_password_reset_email(user, reset_url, - subject_template_name='registration/password_reset_subject.txt', # noqa - email_template_name='api_password_reset_email.html', # noqa - token_generator=default_token_generator, - email_subject=None): +def get_password_reset_email( + user, + reset_url, + subject_template_name="registration/password_reset_subject.txt", # noqa + email_template_name="api_password_reset_email.html", # noqa + token_generator=default_token_generator, + email_subject=None, +): """Creates the subject and email body for password reset email.""" result = urlparse(reset_url) site_name = domain = result.hostname - encoded_username = urlsafe_base64_encode(b(user.username.encode('utf-8'))) + encoded_username = urlsafe_base64_encode(b(user.username.encode("utf-8"))) c = { - 'email': user.email, - 'domain': domain, - 'path': result.path, - 'site_name': site_name, - 'uid': urlsafe_base64_encode(force_bytes(user.pk)), - 'username': user.username, - 'encoded_username': encoded_username, - 'token': token_generator.make_token(user), - 'protocol': result.scheme if result.scheme != '' else 'http', + "email": user.email, + "domain": domain, + "path": result.path, + "site_name": site_name, + "uid": urlsafe_base64_encode(force_bytes(user.pk)), + "username": user.username, + "encoded_username": encoded_username, + "token": token_generator.make_token(user), + "protocol": result.scheme if result.scheme != "" else "http", } # if subject email provided don't load the subject template - subject = email_subject or loader.render_to_string(subject_template_name, - c) + subject = email_subject or loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines - subject = ''.join(subject.splitlines()) + subject = "".join(subject.splitlines()) email = loader.render_to_string(email_template_name, c) return subject, email @@ -66,7 +77,7 @@ def get_user_from_uid(uid): uid = urlsafe_base64_decode(uid) user = User.objects.get(pk=uid) except (TypeError, ValueError, OverflowError, User.DoesNotExist): - raise serializers.ValidationError(_(u"Invalid uid %s") % uid) + raise serializers.ValidationError(_("Invalid uid %s") % uid) return user @@ -91,11 +102,13 @@ def __init__(self, email, reset_url, email_subject=None): self.reset_url = reset_url self.email_subject = email_subject - def save(self, - subject_template_name='registration/password_reset_subject.txt', - email_template_name='api_password_reset_email.html', - token_generator=default_token_generator, - from_email=None): + def save( + self, + subject_template_name="registration/password_reset_subject.txt", + email_template_name="api_password_reset_email.html", + token_generator=default_token_generator, + from_email=None, + ): """ Generates a one-use only link for resetting password and sends to the user. @@ -110,8 +123,12 @@ def save(self, if not user.has_usable_password(): continue subject, email = get_password_reset_email( - user, reset_url, subject_template_name, email_template_name, - email_subject=self.email_subject) + user, + reset_url, + subject_template_name, + email_template_name, + email_subject=self.email_subject, + ) send_mail(subject, email, from_email, [user.email]) @@ -119,17 +136,17 @@ def save(self, class PasswordResetSerializer(serializers.Serializer): email = serializers.EmailField(label=_("Email"), max_length=254) reset_url = serializers.URLField(label=_("Reset URL"), max_length=254) - email_subject = serializers.CharField(label=_("Email Subject"), - required=False, max_length=78, - allow_blank=True) + email_subject = serializers.CharField( + label=_("Email Subject"), required=False, max_length=78, allow_blank=True + ) def validate_email(self, value): users = User.objects.filter(email__iexact=value) if users.count() == 0: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + raise serializers.ValidationError( + _("User '%(value)s' does not exist." % {"value": value}) + ) return value @@ -157,8 +174,8 @@ def validate_uid(self, value): return value def validate(self, attrs): - user = get_user_from_uid(attrs.get('uid')) - token = attrs.get('token') + user = get_user_from_uid(attrs.get("uid")) + token = attrs.get("token") if not default_token_generator.check_token(user, token): raise serializers.ValidationError(_("Invalid token: %s") % token) diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index 1d00d673e9..d9f3f671cd 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -2,7 +2,7 @@ """ Project Serializer module. """ -from future.utils import listvalues +from six import itervalues from django.conf import settings from django.contrib.auth.models import User @@ -13,20 +13,33 @@ from rest_framework import serializers from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (get_organization_members_team, - get_or_create_organization_owners_team) +from onadata.apps.api.tools import ( + get_organization_members_team, + get_or_create_organization_owners_team, +) from onadata.apps.logger.models import Project, XForm from onadata.libs.permissions import ( - OwnerRole, ReadOnlyRole, ManagerRole, get_role, is_organization) -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer + OwnerRole, + ReadOnlyRole, + ManagerRole, + get_role, + is_organization, +) +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, - PROJECT_LINKED_DATAVIEWS, PROJ_OWNER_CACHE, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_PERM_CACHE, + PROJ_SUB_DATE_CACHE, + PROJ_TEAM_USERS_CACHE, + PROJECT_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) from onadata.libs.utils.decorators import check_obj @@ -35,8 +48,11 @@ def get_project_xforms(project): Returns an XForm queryset from project. The prefetched `xforms_prefetch` or `xform_set.filter()` queryset. """ - return (project.xforms_prefetch if hasattr(project, 'xforms_prefetch') else - project.xform_set.filter(deleted_at__isnull=True)) + return ( + project.xforms_prefetch + if hasattr(project, "xforms_prefetch") + else project.xform_set.filter(deleted_at__isnull=True) + ) @check_obj @@ -46,20 +62,17 @@ def get_last_submission_date(project): :param project: The project to find the last submission date for. """ - last_submission_date = cache.get( - '{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk)) + last_submission_date = cache.get("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk)) if last_submission_date: return last_submission_date xforms = get_project_xforms(project) dates = [ - x.last_submission_time for x in xforms - if x.last_submission_time is not None + x.last_submission_time for x in xforms if x.last_submission_time is not None ] dates.sort(reverse=True) last_submission_date = dates[0] if dates else None - cache.set('{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk), - last_submission_date) + cache.set("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk), last_submission_date) return last_submission_date @@ -70,12 +83,12 @@ def get_num_datasets(project): :param project: The project to find datasets for. """ - count = cache.get('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk)) + count = cache.get("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk)) if count: return count count = len(get_project_xforms(project)) - cache.set('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk), count) + cache.set("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk), count) return count @@ -91,8 +104,8 @@ def get_team_permissions(team, project): Return team permissions. """ return project.projectgroupobjectpermission_set.filter( - group__pk=team.pk).values_list( - 'permission__codename', flat=True) + group__pk=team.pk + ).values_list("permission__codename", flat=True) @check_obj @@ -100,7 +113,7 @@ def get_teams(project): """ Return the teams with access to the project. """ - teams_users = cache.get('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk)) + teams_users = cache.get("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk)) if teams_users: return teams_users @@ -112,13 +125,11 @@ def get_teams(project): users = [user.username for user in team.user_set.all()] perms = get_team_permissions(team, project) - teams_users.append({ - "name": team.name, - "role": get_role(perms, project), - "users": users - }) + teams_users.append( + {"name": team.name, "role": get_role(perms, project), "users": users} + ) - cache.set('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) + cache.set("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) return teams_users @@ -128,22 +139,23 @@ def get_users(project, context, all_perms=True): Return a list of users and organizations that have access to the project. """ if all_perms: - users = cache.get('{}{}'.format(PROJ_PERM_CACHE, project.pk)) + users = cache.get("{}{}".format(PROJ_PERM_CACHE, project.pk)) if users: return users data = {} - request_user = context['request'].user + request_user = context["request"].user if not request_user.is_anonymous: request_user_perms = [ perm.permission.codename for perm in project.projectuserobjectpermission_set.filter( - user=request_user)] + user=request_user + ) + ] request_user_role = get_role(request_user_perms, project) - request_user_is_admin = request_user_role in [ - OwnerRole.name, ManagerRole.name] + request_user_is_admin = request_user_role in [OwnerRole.name, ManagerRole.name] else: request_user_is_admin = False @@ -151,29 +163,31 @@ def get_users(project, context, all_perms=True): if perm.user_id not in data: user = perm.user - if all_perms or user in [ - request_user, project.organization - ] or request_user_is_admin: + if ( + all_perms + or user in [request_user, project.organization] + or request_user_is_admin + ): data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append(perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], project) - del data[k]['permissions'] + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], project) + del data[k]["permissions"] - results = listvalues(data) + results = list(itervalues(data)) if all_perms: - cache.set('{}{}'.format(PROJ_PERM_CACHE, project.pk), results) + cache.set("{}{}".format(PROJ_PERM_CACHE, project.pk), results) return results @@ -187,61 +201,77 @@ class BaseProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectXFormSerializer class. """ - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'is_merged_dataset') + fields = ("name", "formid", "id_string", "is_merged_dataset") class ProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ ProjectXFormSerializer class - to return project xform info. """ + url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + view_name="xform-detail", lookup_field="pk" + ) + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") published_by_formbuilder = serializers.SerializerMethodField() class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'num_of_submissions', - 'downloadable', 'encrypted', 'published_by_formbuilder', - 'last_submission_time', 'date_created', 'url', - 'last_updated_at', 'is_merged_dataset', ) + fields = ( + "name", + "formid", + "id_string", + "num_of_submissions", + "downloadable", + "encrypted", + "published_by_formbuilder", + "last_submission_time", + "date_created", + "url", + "last_updated_at", + "is_merged_dataset", + ) def get_published_by_formbuilder(self, obj): # pylint: disable=no-self-use """ Returns true if the form was published by formbuilder. """ - metadata = obj.metadata_set.filter( - data_type='published_by_formbuilder').first() - return (metadata and hasattr(metadata, 'data_value') - and metadata.data_value) + metadata = obj.metadata_set.filter(data_type="published_by_formbuilder").first() + return metadata and hasattr(metadata, "data_value") and metadata.data_value class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectSerializer class. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -250,25 +280,39 @@ class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project fields = [ - 'url', 'projectid', 'owner', 'created_by', 'metadata', 'starred', - 'users', 'forms', 'public', 'tags', 'num_datasets', - 'last_submission_date', 'teams', 'name', 'date_created', - 'date_modified', 'deleted_at' + "url", + "projectid", + "owner", + "created_by", + "metadata", + "starred", + "users", + "forms", + "public", + "tags", + "num_datasets", + "last_submission_date", + "teams", + "name", + "date_created", + "date_modified", + "deleted_at", ] def get_starred(self, obj): """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_users(self, obj): """ Return a list of users and organizations that have access to the project. """ - owner_query_param_in_request = 'request' in self.context and\ - "owner" in self.context['request'].GET + owner_query_param_in_request = ( + "request" in self.context and "owner" in self.context["request"].GET + ) return get_users(obj, self.context, owner_query_param_in_request) @check_obj @@ -276,16 +320,17 @@ def get_forms(self, obj): """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk)) + forms = cache.get("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk)) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = BaseProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) + cache.set("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) return forms @@ -312,11 +357,12 @@ def can_add_project_to_profile(user, organization): """ Check if user has permission to add a project to a profile. """ - perms = 'can_add_project' - if user != organization and \ - not user.has_perm(perms, organization.profile) and \ - not user.has_perm( - perms, OrganizationProfile.objects.get(user=organization)): + perms = "can_add_project" + if ( + user != organization + and not user.has_perm(perms, organization.profile) + and not user.has_perm(perms, OrganizationProfile.objects.get(user=organization)) + ): return False return True @@ -326,22 +372,27 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): """ ProjectSerializer class - creates and updates a project. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -350,21 +401,22 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project - exclude = ('shared', 'user_stars', 'deleted_by', 'organization') + exclude = ("shared", "user_stars", "deleted_by", "organization") def validate(self, attrs): - name = attrs.get('name') - organization = attrs.get('organization') + name = attrs.get("name") + organization = attrs.get("organization") if not self.instance and organization: project_w_same_name = Project.objects.filter( - name__iexact=name, - organization=organization) + name__iexact=name, organization=organization + ) if project_w_same_name: - raise serializers.ValidationError({ - 'name': _(u"Project {} already exists.".format(name))}) + raise serializers.ValidationError( + {"name": _("Project {} already exists.".format(name))} + ) else: organization = organization or self.instance.organization - request = self.context['request'] + request = self.context["request"] try: has_perm = can_add_project_to_profile(request.user, organization) except OrganizationProfile.DoesNotExist: @@ -372,12 +424,15 @@ def validate(self, attrs): # A user does not require permissions to the user's account forms. has_perm = False if not has_perm: - raise serializers.ValidationError({ - 'owner': - _("You do not have permission to create a project " - "in the organization %(organization)s." % { - 'organization': organization}) - }) + raise serializers.ValidationError( + { + "owner": _( + "You do not have permission to create a project " + "in the organization %(organization)s." + % {"organization": organization} + ) + } + ) return attrs def validate_public(self, value): # pylint: disable=no-self-use @@ -386,7 +441,8 @@ def validate_public(self, value): # pylint: disable=no-self-use """ if not settings.ALLOW_PUBLIC_DATASETS and value: raise serializers.ValidationError( - _('Public projects are currently disabled.')) + _("Public projects are currently disabled.") + ) return value def validate_metadata(self, value): # pylint: disable=no-self-use @@ -404,33 +460,32 @@ def validate_metadata(self, value): # pylint: disable=no-self-use return value def update(self, instance, validated_data): - metadata = JsonField.to_json(validated_data.get('metadata')) + metadata = JsonField.to_json(validated_data.get("metadata")) if metadata is None: metadata = dict() - owner = validated_data.get('organization') + owner = validated_data.get("organization") if self.partial and metadata: if not isinstance(instance.metadata, dict): instance.metadata = {} instance.metadata.update(metadata) - validated_data['metadata'] = instance.metadata + validated_data["metadata"] = instance.metadata if self.partial and owner: # give the new owner permissions set_owners_permission(owner, instance) if is_organization(owner.profile): - owners_team = get_or_create_organization_owners_team( - owner.profile) + owners_team = get_or_create_organization_owners_team(owner.profile) members_team = get_organization_members_team(owner.profile) OwnerRole.add(owners_team, instance) ReadOnlyRole.add(members_team, instance) owners = owners_team.user_set.all() # Owners are also members members = members_team.user_set.exclude( - username__in=[ - user.username for user in owners]) + username__in=[user.username for user in owners] + ) # Exclude new owner if in members members = members.exclude(username=owner.username) @@ -439,47 +494,50 @@ def update(self, instance, validated_data): [ReadOnlyRole.add(member, instance) for member in members] # clear cache - safe_delete('{}{}'.format(PROJ_PERM_CACHE, instance.pk)) + safe_delete("{}{}".format(PROJ_PERM_CACHE, instance.pk)) - project = super(ProjectSerializer, self)\ - .update(instance, validated_data) + project = super(ProjectSerializer, self).update(instance, validated_data) - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) return instance @track_object_event( - user_field='created_by', + user_field="created_by", properties={ - 'created_by': 'created_by', - 'project_id': 'pk', - 'project_name': 'name'} + "created_by": "created_by", + "project_id": "pk", + "project_name": "name", + }, ) def create(self, validated_data): - metadata = validated_data.get('metadata', dict()) + metadata = validated_data.get("metadata", dict()) if metadata is None: metadata = dict() - created_by = self.context['request'].user + created_by = self.context["request"].user try: project = Project.objects.create( # pylint: disable=E1101 - name=validated_data.get('name'), - organization=validated_data.get('organization'), + name=validated_data.get("name"), + organization=validated_data.get("organization"), created_by=created_by, - shared=validated_data.get('shared', False), - metadata=metadata) + shared=validated_data.get("shared", False), + metadata=metadata, + ) except IntegrityError: raise serializers.ValidationError( - "The fields name, organization must make a unique set.") + "The fields name, organization must make a unique set." + ) else: - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) - request = self.context.get('request') - serializer = ProjectSerializer( - project, context={'request': request}) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) + request = self.context.get("request") + serializer = ProjectSerializer(project, context={"request": request}) response = serializer.data - cache.set(f'{PROJ_OWNER_CACHE}{project.pk}', response) + cache.set(f"{PROJ_OWNER_CACHE}{project.pk}", response) return project def get_users(self, obj): # pylint: disable=no-self-use @@ -494,15 +552,16 @@ def get_forms(self, obj): # pylint: disable=no-self-use """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_FORMS_CACHE, obj.pk)) + forms = cache.get("{}{}".format(PROJ_FORMS_CACHE, obj.pk)) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = ProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_FORMS_CACHE, obj.pk), forms) + cache.set("{}{}".format(PROJ_FORMS_CACHE, obj.pk), forms) return forms @@ -522,7 +581,7 @@ def get_starred(self, obj): # pylint: disable=no-self-use """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_teams(self, obj): # pylint: disable=no-self-use """ @@ -535,18 +594,21 @@ def get_data_views(self, obj): """ Return a list of filtered datasets. """ - data_views = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk)) + data_views = cache.get("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk)) if data_views: return data_views - data_views_obj = obj.dataview_prefetch if \ - hasattr(obj, 'dataview_prefetch') else\ - obj.dataview_set.filter(deleted_at__isnull=True) + data_views_obj = ( + obj.dataview_prefetch + if hasattr(obj, "dataview_prefetch") + else obj.dataview_set.filter(deleted_at__isnull=True) + ) serializer = DataViewMinimalSerializer( - data_views_obj, many=True, context=self.context) + data_views_obj, many=True, context=self.context + ) data_views = list(serializer.data) - cache.set('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) + cache.set("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) return data_views diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index e93fb24ce9..5e99fa0ffa 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -2,7 +2,7 @@ from django.http import Http404 from django.urls import resolve, get_script_prefix, Resolver404 from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from guardian.shortcuts import get_users_with_perms from rest_framework import serializers @@ -19,32 +19,32 @@ class GenericRelatedField(serializers.HyperlinkedRelatedField): default_error_messages = { - 'incorrect_match': _('`{input}` is not a valid relation.') + "incorrect_match": _("`{input}` is not a valid relation.") } def __init__(self, *args, **kwargs): - self.view_names = ['xform-detail', 'dataviews-detail'] + self.view_names = ["xform-detail", "dataviews-detail"] self.resolve = resolve self.reverse = reverse - self.format = kwargs.pop('format', 'json') + self.format = kwargs.pop("format", "json") super(serializers.RelatedField, self).__init__(*args, **kwargs) def _setup_field(self, view_name): self.lookup_url_kwarg = self.lookup_field - if view_name == 'xform-detail': + if view_name == "xform-detail": self.queryset = XForm.objects.all() - if view_name == 'dataviews-detail': + if view_name == "dataviews-detail": self.queryset = DataView.objects.all() def to_representation(self, value): if isinstance(value, XForm): - self.view_name = 'xform-detail' + self.view_name = "xform-detail" elif isinstance(value, DataView): - self.view_name = 'dataviews-detail' + self.view_name = "dataviews-detail" else: - raise Exception(_(u"Unknown type for content_object.")) + raise Exception(_("Unknown type for content_object.")) self._setup_field(self.view_name) @@ -52,31 +52,31 @@ def to_representation(self, value): def to_internal_value(self, data): try: - http_prefix = data.startswith(('http:', 'https:')) + http_prefix = data.startswith(("http:", "https:")) except AttributeError: - self.fail('incorrect_type', data_type=type(data).__name__) + self.fail("incorrect_type", data_type=type(data).__name__) input_data = data if http_prefix: # If needed convert absolute URLs to relative path data = urlparse(data).path prefix = get_script_prefix() if data.startswith(prefix): - data = '/' + data[len(prefix):] + data = "/" + data[len(prefix) :] try: match = self.resolve(data) except Resolver404: - self.fail('no_match') + self.fail("no_match") if match.view_name not in self.view_names: - self.fail('incorrect_match', input=input_data) + self.fail("incorrect_match", input=input_data) self._setup_field(match.view_name) try: return self.get_object(match.view_name, match.args, match.kwargs) except (ObjectDoesNotExist, TypeError, ValueError): - self.fail('does_not_exist') + self.fail("does_not_exist") return data @@ -84,8 +84,7 @@ def to_internal_value(self, data): class WidgetSerializer(serializers.HyperlinkedModelSerializer): id = serializers.ReadOnlyField() url = serializers.HyperlinkedIdentityField( - view_name='widgets-detail', - lookup_field='pk' + view_name="widgets-detail", lookup_field="pk" ) content_object = GenericRelatedField() key = serializers.CharField(read_only=True) @@ -95,17 +94,30 @@ class WidgetSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Widget - fields = ('id', 'url', 'key', 'title', 'description', 'widget_type', - 'order', 'view_type', 'column', 'group_by', 'content_object', - 'data', 'aggregation', 'metadata') + fields = ( + "id", + "url", + "key", + "title", + "description", + "widget_type", + "order", + "view_type", + "column", + "group_by", + "content_object", + "data", + "aggregation", + "metadata", + ) def get_data(self, obj): # Get the request obj - request = self.context.get('request') + request = self.context.get("request") # Check if data flag is present - data_flag = request.GET.get('data') - key = request.GET.get('key') + data_flag = request.GET.get("data") + key = request.GET.get("key") if (str2bool(data_flag) or key) and obj: data = Widget.query_data(obj) @@ -115,11 +127,11 @@ def get_data(self, obj): return data def validate(self, attrs): - column = attrs.get('column') + column = attrs.get("column") # Get the form - if 'content_object' in attrs: - content_object = attrs.get('content_object') + if "content_object" in attrs: + content_object = attrs.get("content_object") if isinstance(content_object, XForm): xform = content_object @@ -131,11 +143,11 @@ def validate(self, attrs): # Check if column exists in xform get_field_from_field_xpath(column, xform) except Http404: - raise serializers.ValidationError({ - 'column': (u"'{}' not in the form.".format(column)) - }) + raise serializers.ValidationError( + {"column": ("'{}' not in the form.".format(column))} + ) - order = attrs.get('order') + order = attrs.get("order") # Set the order if order: @@ -144,19 +156,20 @@ def validate(self, attrs): return attrs def validate_content_object(self, value): - request = self.context.get('request') + request = self.context.get("request") users = get_users_with_perms( value.project, attach_perms=False, with_group_users=False ) profile = value.project.organization.profile # Shared or an admin in the organization - if request.user not in users and not\ - is_organization(profile) and not\ - OwnerRole.user_has_role(request.user, - profile): - raise serializers.ValidationError(_( - u"You don't have permission to the Project." - )) + if ( + request.user not in users + and not is_organization(profile) + and not OwnerRole.user_has_role(request.user, profile) + ): + raise serializers.ValidationError( + _("You don't have permission to the Project.") + ) return value diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 3ab8fcfd13..f886ed6726 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -11,8 +11,8 @@ from django.core.validators import URLValidator from django.db.models import Count from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import listvalues +from six.moves.urllib.parse import urlparse +from six import itervalues from requests.exceptions import ConnectionError from rest_framework import serializers from rest_framework.reverse import reverse @@ -22,24 +22,27 @@ from onadata.apps.logger.models import XFormVersion from onadata.libs.exceptions import EnketoError from onadata.libs.permissions import get_role, is_organization -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.metadata_serializer import MetaDataSerializer from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.cache_tools import ( - ENKETO_PREVIEW_URL_CACHE, ENKETO_URL_CACHE, ENKETO_SINGLE_SUBMIT_URL_CACHE, - XFORM_LINKED_DATAVIEWS, XFORM_METADATA_CACHE, XFORM_PERMISSIONS_CACHE, - XFORM_DATA_VERSIONS, XFORM_COUNT) -from onadata.libs.utils.common_tags import (GROUP_DELIMETER_TAG, - REPEAT_INDEX_TAGS) + ENKETO_PREVIEW_URL_CACHE, + ENKETO_URL_CACHE, + ENKETO_SINGLE_SUBMIT_URL_CACHE, + XFORM_LINKED_DATAVIEWS, + XFORM_METADATA_CACHE, + XFORM_PERMISSIONS_CACHE, + XFORM_DATA_VERSIONS, + XFORM_COUNT, +) +from onadata.libs.utils.common_tags import GROUP_DELIMETER_TAG, REPEAT_INDEX_TAGS from onadata.libs.utils.decorators import check_obj -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) def _create_enketo_urls(request, xform): @@ -50,24 +53,27 @@ def _create_enketo_urls(request, xform): :param xform: :return: enketo urls """ - form_url = get_form_url(request, xform.user.username, - settings.ENKETO_PROTOCOL, xform_pk=xform.pk, - generate_consistent_urls=True) + form_url = get_form_url( + request, + xform.user.username, + settings.ENKETO_PROTOCOL, + xform_pk=xform.pk, + generate_consistent_urls=True, + ) data = {} try: enketo_urls = get_enketo_urls(form_url, xform.id_string) offline_url = enketo_urls.get("offline_url") MetaData.enketo_url(xform, offline_url) - data['offline_url'] = offline_url - if 'preview_url' in enketo_urls: + data["offline_url"] = offline_url + if "preview_url" in enketo_urls: preview_url = enketo_urls.get("preview_url") MetaData.enketo_preview_url(xform, preview_url) - data['preview_url'] = preview_url - if 'single_url' in enketo_urls: + data["preview_url"] = preview_url + if "single_url" in enketo_urls: single_url = enketo_urls.get("single_url") - MetaData.enketo_single_submit_url( - xform, single_url) - data['single_url'] = single_url + MetaData.enketo_single_submit_url(xform, single_url) + data["single_url"] = single_url except ConnectionError as e: logging.exception("Connection Error: %s" % e) except EnketoError as e: @@ -85,22 +91,26 @@ def _set_cache(cache_key, cache_data, obj): :param obj: :return: Data that has been cached """ - cache.set('{}{}'.format(cache_key, obj.pk), cache_data) + cache.set("{}{}".format(cache_key, obj.pk), cache_data) return cache_data def user_to_username(item): - item['user'] = item['user'].username + item["user"] = item["user"].username return item def clean_public_key(value): - if value.startswith('-----BEGIN PUBLIC KEY-----') and\ - value.endswith('-----END PUBLIC KEY-----'): - return value.replace('-----BEGIN PUBLIC KEY-----', - '').replace('-----END PUBLIC KEY-----', - '').replace(' ', '').rstrip() + if value.startswith("-----BEGIN PUBLIC KEY-----") and value.endswith( + "-----END PUBLIC KEY-----" + ): + return ( + value.replace("-----BEGIN PUBLIC KEY-----", "") + .replace("-----END PUBLIC KEY-----", "") + .replace(" ", "") + .rstrip() + ) class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): @@ -109,27 +119,27 @@ class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): Credits: https://stackoverflow.com/a/31161585 """ - lookup_fields = (('pk', 'pk'),) + + lookup_fields = (("pk", "pk"),) def __init__(self, *args, **kwargs): - self.lookup_fields = kwargs.pop('lookup_fields', self.lookup_fields) + self.lookup_fields = kwargs.pop("lookup_fields", self.lookup_fields) super(MultiLookupIdentityField, self).__init__(*args, **kwargs) def get_url(self, obj, view_name, request, format): kwargs = {} for model_field, url_param in self.lookup_fields: attr = obj - for field in model_field.split('__'): + for field in model_field.split("__"): attr = getattr(attr, field) kwargs[url_param] = attr - if not format and hasattr(self, 'format'): + if not format and hasattr(self, "format"): fmt = self.format else: fmt = format - return reverse( - view_name, kwargs=kwargs, request=request, format=fmt) + return reverse(view_name, kwargs=kwargs, request=request, format=fmt) class XFormMixin(object): @@ -153,52 +163,48 @@ def _get_metadata(self, obj, key): def get_users(self, obj): xform_perms = [] if obj: - xform_perms = cache.get( - '{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk)) + xform_perms = cache.get("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk)) if xform_perms: return xform_perms - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), - xform_perms) + cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) data = {} for perm in obj.xformuserobjectpermission_set.all(): if perm.user_id not in data: user = perm.user data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append( - perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], XForm) - del (data[k]['permissions']) + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], XForm) + del data[k]["permissions"] - xform_perms = listvalues(data) + xform_perms = list(itervalues(data)) - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) + cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) return xform_perms def get_enketo_url(self, obj): if obj: - _enketo_url = cache.get('{}{}'.format(ENKETO_URL_CACHE, obj.pk)) + _enketo_url = cache.get("{}{}".format(ENKETO_URL_CACHE, obj.pk)) if _enketo_url: return _enketo_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") return _set_cache(ENKETO_URL_CACHE, url, obj) @@ -207,35 +213,33 @@ def get_enketo_url(self, obj): def get_enketo_single_submit_url(self, obj): if obj: _enketo_single_submit_url = cache.get( - '{}{}'.format(ENKETO_SINGLE_SUBMIT_URL_CACHE, - obj.pk)) + "{}{}".format(ENKETO_SINGLE_SUBMIT_URL_CACHE, obj.pk) + ) if _enketo_single_submit_url: return _enketo_single_submit_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") - return _set_cache( - ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) + return _set_cache(ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) return None def get_enketo_preview_url(self, obj): if obj: _enketo_preview_url = cache.get( - '{}{}'.format(ENKETO_PREVIEW_URL_CACHE, obj.pk)) + "{}{}".format(ENKETO_PREVIEW_URL_CACHE, obj.pk) + ) if _enketo_preview_url: return _enketo_preview_url - url = self._get_metadata(obj, 'enketo_preview_url') + url = self._get_metadata(obj, "enketo_preview_url") if url is None: try: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('preview_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("preview_url") except Exception: return url @@ -245,14 +249,16 @@ def get_enketo_preview_url(self, obj): def get_data_views(self, obj): if obj: - key = '{}{}'.format(XFORM_LINKED_DATAVIEWS, obj.pk) + key = "{}{}".format(XFORM_LINKED_DATAVIEWS, obj.pk) data_views = cache.get(key) if data_views: return data_views data_views = DataViewMinimalSerializer( obj.dataview_set.filter(deleted_at__isnull=True), - many=True, context=self.context).data + many=True, + context=self.context, + ).data cache.set(key, list(data_views)) @@ -261,7 +267,7 @@ def get_data_views(self, obj): def get_num_of_submissions(self, obj): if obj: - key = '{}{}'.format(XFORM_COUNT, obj.pk) + key = "{}{}".format(XFORM_COUNT, obj.pk) count = cache.get(key) if count: return count @@ -278,43 +284,49 @@ def get_last_submission_time(self, obj): If a form is a merged dataset then it is picked from the list of forms attached to that merged dataset. """ - if 'last_submission_time' not in self.fields: + if "last_submission_time" not in self.fields: return None if obj.is_merged_dataset: values = [ x.last_submission_time.isoformat() - for x in obj.mergedxform.xforms.only('last_submission_time') + for x in obj.mergedxform.xforms.only("last_submission_time") if x.last_submission_time ] if values: return sorted(values, reverse=True)[0] - return obj.last_submission_time.isoformat() \ - if obj.last_submission_time else None + return ( + obj.last_submission_time.isoformat() if obj.last_submission_time else None + ) class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + formid = serializers.ReadOnlyField(source="id") owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -326,37 +338,58 @@ class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + formid = serializers.ReadOnlyField(source="id") metadata = serializers.SerializerMethodField() owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() submission_count_for_today = serializers.ReadOnlyField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -369,27 +402,42 @@ class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) def get_metadata(self, obj): xform_metadata = [] if obj: - xform_metadata = cache.get( - '{}{}'.format(XFORM_METADATA_CACHE, obj.pk)) + xform_metadata = cache.get("{}{}".format(XFORM_METADATA_CACHE, obj.pk)) if xform_metadata: return xform_metadata xform_metadata = list( MetaDataSerializer( - obj.metadata_set.all(), many=True, context=self.context) - .data) - cache.set('{}{}'.format(XFORM_METADATA_CACHE, obj.pk), - xform_metadata) + obj.metadata_set.all(), many=True, context=self.context + ).data + ) + cache.set("{}{}".format(XFORM_METADATA_CACHE, obj.pk), xform_metadata) return xform_metadata @@ -400,11 +448,11 @@ def validate_public_key(self, value): # pylint: disable=no-self-use package """ try: - load_pem_public_key( - value.encode('utf-8'), backend=default_backend()) + load_pem_public_key(value.encode("utf-8"), backend=default_backend()) except ValueError: raise serializers.ValidationError( - _('The public key is not a valid base64 RSA key')) + _("The public key is not a valid base64 RSA key") + ) return clean_public_key(value) def _check_if_allowed_public(self, value): # pylint: disable=no-self-use @@ -413,8 +461,7 @@ def _check_if_allowed_public(self, value): # pylint: disable=no-self-use forms """ if not settings.ALLOW_PUBLIC_DATASETS and value: - raise serializers.ValidationError( - _('Public forms are currently disabled.')) + raise serializers.ValidationError(_("Public forms are currently disabled.")) return value def validate_public_data(self, value): @@ -432,7 +479,7 @@ def validate_public(self, value): def get_form_versions(self, obj): versions = [] if obj: - versions = cache.get('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk)) + versions = cache.get("{}{}".format(XFORM_DATA_VERSIONS, obj.pk)) if versions: return versions @@ -441,11 +488,12 @@ def get_form_versions(self, obj): versions = list( Instance.objects.filter(xform=obj, deleted_at__isnull=True) - .values('version').annotate(total=Count('version'))) + .values("version") + .annotate(total=Count("version")) + ) if versions: - cache.set('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk), - list(versions)) + cache.set("{}{}".format(XFORM_DATA_VERSIONS, obj.pk), list(versions)) return versions @@ -458,58 +506,59 @@ def get_has_id_string_changed(self, obj): class XFormListSerializer(serializers.Serializer): - formID = serializers.ReadOnlyField(source='id_string') - name = serializers.ReadOnlyField(source='title') + formID = serializers.ReadOnlyField(source="id_string") + name = serializers.ReadOnlyField(source="title") version = serializers.ReadOnlyField() hash = serializers.ReadOnlyField() - descriptionText = serializers.ReadOnlyField(source='description') - downloadUrl = serializers.SerializerMethodField('get_url') - manifestUrl = serializers.SerializerMethodField('get_manifest_url') + descriptionText = serializers.ReadOnlyField(source="description") + downloadUrl = serializers.SerializerMethodField("get_url") + manifestUrl = serializers.SerializerMethodField("get_manifest_url") @check_obj def get_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") - return reverse('download_xform', kwargs=kwargs, request=request) + return reverse("download_xform", kwargs=kwargs, request=request) @check_obj def get_manifest_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') - object_list = MetaData.objects.filter(data_type='media', - object_id=obj.pk) + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") + object_list = MetaData.objects.filter(data_type="media", object_id=obj.pk) if object_list: - return reverse('manifest-url', kwargs=kwargs, request=request) + return reverse("manifest-url", kwargs=kwargs, request=request) return None class XFormManifestSerializer(serializers.Serializer): filename = serializers.SerializerMethodField() hash = serializers.SerializerMethodField() - downloadUrl = serializers.SerializerMethodField('get_url') + downloadUrl = serializers.SerializerMethodField("get_url") @check_obj def get_url(self, obj): kwargs = { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, } - request = self.context.get('request') + request = self.context.get("request") try: - fmt = obj.data_value[obj.data_value.rindex('.') + 1:] + fmt = obj.data_value[obj.data_value.rindex(".") + 1 :] except ValueError: - fmt = 'csv' - url = reverse( - 'xform-media', kwargs=kwargs, request=request, format=fmt.lower()) + fmt = "csv" + url = reverse("xform-media", kwargs=kwargs, request=request, format=fmt.lower()) group_delimiter = self.context.get(GROUP_DELIMETER_TAG) repeat_index_tags = self.context.get(REPEAT_INDEX_TAGS) - if group_delimiter and repeat_index_tags and fmt == 'csv': - return (url+"?%s=%s&%s=%s" % ( - GROUP_DELIMETER_TAG, group_delimiter, REPEAT_INDEX_TAGS, - repeat_index_tags)) + if group_delimiter and repeat_index_tags and fmt == "csv": + return url + "?%s=%s&%s=%s" % ( + GROUP_DELIMETER_TAG, + group_delimiter, + REPEAT_INDEX_TAGS, + repeat_index_tags, + ) return url @@ -517,38 +566,42 @@ def get_url(self, obj): def get_hash(self, obj): filename = obj.data_value hsh = obj.file_hash - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", xform pk is the # second item # other file uploads other than linked datasets have a data_file - if len(parts) > 2 and obj.data_file == '': + if len(parts) > 2 and obj.data_file == "": dataset_type = parts[0] pk = parts[1] xform = None - if dataset_type == 'xform': - xform = XForm.objects.filter(pk=pk)\ - .only('last_submission_time').first() + if dataset_type == "xform": + xform = XForm.objects.filter(pk=pk).only("last_submission_time").first() else: - data_view = DataView.objects.filter(pk=pk)\ - .only('xform__last_submission_time').first() + data_view = ( + DataView.objects.filter(pk=pk) + .only("xform__last_submission_time") + .first() + ) if data_view: xform = data_view.xform if xform and xform.last_submission_time: - hsh = u'md5:%s' % (md5( - xform.last_submission_time.isoformat().encode( - 'utf-8')).hexdigest()) + hsh = "md5:%s" % ( + md5( + xform.last_submission_time.isoformat().encode("utf-8") + ).hexdigest() + ) - return u"%s" % (hsh or 'md5:') + return "%s" % (hsh or "md5:") @check_obj def get_filename(self, obj): filename = obj.data_value - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = u'%s.csv' % parts[2] + filename = "%s.csv" % parts[2] else: try: URLValidator()(filename) @@ -563,25 +616,27 @@ def get_filename(self, obj): class XFormVersionListSerializer(serializers.ModelSerializer): xform = serializers.HyperlinkedRelatedField( - view_name='xform-detail', lookup_field='pk', - queryset=XForm.objects.filter(deleted_at__isnull=True) + view_name="xform-detail", + lookup_field="pk", + queryset=XForm.objects.filter(deleted_at__isnull=True), ) url = MultiLookupIdentityField( - view_name='form-version-detail', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) xml = MultiLookupIdentityField( - view_name='form-version-detail', - format='xml', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + format="xml", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), ) class Meta: model = XFormVersion - exclude = ('json', 'xls', 'id') + exclude = ("json", "xls", "id") diff --git a/onadata/libs/tests/utils/test_email.py b/onadata/libs/tests/utils/test_email.py index d1f643c512..5236df217d 100644 --- a/onadata/libs/tests/utils/test_email.py +++ b/onadata/libs/tests/utils/test_email.py @@ -1,77 +1,83 @@ -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.test import RequestFactory from django.test.utils import override_settings from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.email import ( - get_verification_email_data, get_verification_url -) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url VERIFICATION_URL = "http://ab.cd.ef" class TestEmail(TestBase): - def setUp(self): self.email = "john@doe.com" - self.username = "johndoe", + self.username = ("johndoe",) self.verification_key = "123abc" self.redirect_url = "http://red.ir.ect" - self.custom_request = RequestFactory().get( - '/path', data={'name': u'test'} - ) + self.custom_request = RequestFactory().get("/path", data={"name": "test"}) @override_settings(VERIFICATION_URL=None) def test_get_verification_url(self): # without redirect_url - verification_url = get_verification_url(**{ - "redirect_url": None, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": None, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), ) # with redirect_url - verification_url = get_verification_url(**{ - "redirect_url": self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) - string_query_params = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) + string_query_params = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?%s' - % string_query_params) + ("http://testserver/api/v1/profiles/verify_email?%s" % string_query_params), ) def _get_email_data(self, include_redirect_url=False): - verification_url = get_verification_url(**{ - "redirect_url": include_redirect_url and self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) - - email_data = get_verification_email_data(**{ - "email": self.email, - "username": self.username, - "verification_url": verification_url, - "request": self.custom_request - }) - - self.assertIn('email', email_data) - self.assertIn(self.email, email_data.get('email')) - self.assertIn('subject', email_data) - self.assertIn('message_txt', email_data) + verification_url = get_verification_url( + **{ + "redirect_url": include_redirect_url and self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) + + email_data = get_verification_email_data( + **{ + "email": self.email, + "username": self.username, + "verification_url": verification_url, + "request": self.custom_request, + } + ) + + self.assertIn("email", email_data) + self.assertIn(self.email, email_data.get("email")) + self.assertIn("subject", email_data) + self.assertIn("message_txt", email_data) return email_data @@ -79,33 +85,32 @@ def _get_email_data(self, include_redirect_url=False): def test_get_verification_email_data_without_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), - email_data.get('message_txt') + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) def test_get_verification_email_data_with_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - '{}?verification_key={}'.format( - VERIFICATION_URL, self.verification_key - ), - email_data.get('message_txt') + "{}?verification_key={}".format(VERIFICATION_URL, self.verification_key), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) - def test_get_verification_email_data_with_verification_and_redirect_urls( - self): + def test_get_verification_email_data_with_verification_and_redirect_urls(self): email_data = self._get_email_data(include_redirect_url=True) - encoded_url = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) - self.assertIn( - encoded_url.replace('&', '&'), email_data.get('message_txt') + encoded_url = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } ) + self.assertIn(encoded_url.replace("&", "&"), email_data.get("message_txt")) def test_email_data_does_not_contain_newline_chars(self): email_data = self._get_email_data(include_redirect_url=True) - self.assertNotIn('\n', email_data.get('subject')) + self.assertNotIn("\n", email_data.get("subject")) diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index f3f30e576e..ee955369c1 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -4,7 +4,7 @@ from io import StringIO from xml.parsers.expat import ExpatError -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.core.files.base import ContentFile from django.core.files.storage import default_storage @@ -17,8 +17,7 @@ from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.libs.utils.common_tools import retry -from onadata.libs.utils.logger_tools import (PublishXForm, create_instance, - publish_form) +from onadata.libs.utils.logger_tools import PublishXForm, create_instance, publish_form NUM_RETRIES = 3 @@ -30,7 +29,7 @@ def django_file(file_obj, field_name, content_type): name=file_obj.name, content_type=content_type, size=file_obj.size, - charset=None + charset=None, ) @@ -49,12 +48,12 @@ def _get_form_list(xml_text): forms = [] for childNode in xml_doc.childNodes: - if childNode.nodeName == 'xforms': + if childNode.nodeName == "xforms": for xformNode in childNode.childNodes: - if xformNode.nodeName == 'xform': - id_string = node_value(xformNode, 'formID') - download_url = node_value(xformNode, 'downloadUrl') - manifest_url = node_value(xformNode, 'manifestUrl') + if xformNode.nodeName == "xform": + id_string = node_value(xformNode, "formID") + download_url = node_value(xformNode, "downloadUrl") + manifest_url = node_value(xformNode, "manifestUrl") forms.append((id_string, download_url, manifest_url)) return forms @@ -64,8 +63,8 @@ def _get_instances_uuids(xml_doc): uuids = [] for child_node in xml_doc.childNodes: - if child_node.nodeName == 'idChunk': - for id_node in child_node.getElementsByTagName('id'): + if child_node.nodeName == "idChunk": + for id_node in child_node.getElementsByTagName("id"): if id_node.childNodes: uuid = id_node.childNodes[0].nodeValue uuids.append(uuid) @@ -78,14 +77,12 @@ def __init__(self, url, username, password, user): self.url = url self.user = user self.auth = HTTPDigestAuth(username, password) - self.form_list_url = urljoin(self.url, 'formList') - self.submission_list_url = urljoin(self.url, 'view/submissionList') - self.download_submission_url = urljoin(self.url, - 'view/downloadSubmission') - self.forms_path = os.path.join( - self.user.username, 'briefcase', 'forms') + self.form_list_url = urljoin(self.url, "formList") + self.submission_list_url = urljoin(self.url, "view/submissionList") + self.download_submission_url = urljoin(self.url, "view/downloadSubmission") + self.forms_path = os.path.join(self.user.username, "briefcase", "forms") self.resumption_cursor = 0 - self.logger = logging.getLogger('console_logger') + self.logger = logging.getLogger("console_logger") def download_manifest(self, manifest_url, id_string): if self._get_response(manifest_url): @@ -96,8 +93,7 @@ def download_manifest(self, manifest_url, id_string): except ExpatError: return - manifest_path = os.path.join( - self.forms_path, id_string, 'form-media') + manifest_path = os.path.join(self.forms_path, id_string, "form-media") self.logger.debug("Downloading media files for %s" % id_string) self.download_media_files(manifest_doc, manifest_path) @@ -105,8 +101,11 @@ def download_manifest(self, manifest_url, id_string): def download_xforms(self, include_instances=False): # fetch formList if not self._get_response(self.form_list_url): - response = self._current_response.content \ - if self._current_response else "Unknown Error" + response = ( + self._current_response.content + if self._current_response + else "Unknown Error" + ) self.logger.error("Failed to download xforms %s." % response) return @@ -114,16 +113,14 @@ def download_xforms(self, include_instances=False): response = self._current_response forms = _get_form_list(response.content) - self.logger.debug('Successfull fetched %s.' % self.form_list_url) + self.logger.debug("Successfull fetched %s." % self.form_list_url) for id_string, download_url, manifest_url in forms: - form_path = os.path.join( - self.forms_path, id_string, '%s.xml' % id_string) + form_path = os.path.join(self.forms_path, id_string, "%s.xml" % id_string) if not default_storage.exists(form_path): if not self._get_response(download_url): - self.logger.error("Failed to download xform %s." - % download_url) + self.logger.error("Failed to download xform %s." % download_url) continue form_res = self._current_response @@ -140,8 +137,7 @@ def download_xforms(self, include_instances=False): if include_instances: self.download_instances(id_string) - self.logger.debug("Done downloading submissions for %s" % - id_string) + self.logger.debug("Done downloading submissions for %s" % id_string) @retry(NUM_RETRIES) def _get_response(self, url, params=None): @@ -159,7 +155,7 @@ def _get_media_response(self, url): # S3 redirects, avoid using formhub digest on S3 if head_response.status_code == 302: - url = head_response.headers.get('location') + url = head_response.headers.get("location") response = requests.get(url) success = response.status_code == 200 @@ -168,9 +164,9 @@ def _get_media_response(self, url): return success def download_media_files(self, xml_doc, media_path): - for media_node in xml_doc.getElementsByTagName('mediaFile'): - filename_node = media_node.getElementsByTagName('filename') - url_node = media_node.getElementsByTagName('downloadUrl') + for media_node in xml_doc.getElementsByTagName("mediaFile"): + filename_node = media_node.getElementsByTagName("filename") + url_node = media_node.getElementsByTagName("downloadUrl") if filename_node and url_node: filename = filename_node[0].childNodes[0].nodeValue path = os.path.join(media_path, filename) @@ -187,37 +183,41 @@ def download_media_files(self, xml_doc, media_path): def download_instances(self, form_id, cursor=0, num_entries=100): self.logger.debug("Starting submissions download for %s" % form_id) - if not self._get_response(self.submission_list_url, - params={'formId': form_id, - 'numEntries': num_entries, - 'cursor': cursor}): - self.logger.error("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + if not self._get_response( + self.submission_list_url, + params={"formId": form_id, "numEntries": num_entries, "cursor": cursor}, + ): + self.logger.error( + "Fetching %s formId: %s, cursor: %s" + % (self.submission_list_url, form_id, cursor) + ) return response = self._current_response - self.logger.debug("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + self.logger.debug( + "Fetching %s formId: %s, cursor: %s" + % (self.submission_list_url, form_id, cursor) + ) try: xml_doc = clean_and_parse_xml(response.content) except ExpatError: return instances = _get_instances_uuids(xml_doc) - path = os.path.join(self.forms_path, form_id, 'instances') + path = os.path.join(self.forms_path, form_id, "instances") for uuid in instances: self.logger.debug("Fetching %s %s submission" % (uuid, form_id)) - form_str = u'%(formId)s[@version=null and @uiVersion=null]/'\ - u'%(formId)s[@key=%(instanceId)s]' % { - 'formId': form_id, - 'instanceId': uuid - } - instance_path = os.path.join(path, uuid.replace(':', ''), - 'submission.xml') + form_str = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=%(instanceId)s]" + % {"formId": form_id, "instanceId": uuid} + ) + instance_path = os.path.join(path, uuid.replace(":", ""), "submission.xml") if not default_storage.exists(instance_path): - if self._get_response(self.download_submission_url, - params={'formId': form_str}): + if self._get_response( + self.download_submission_url, params={"formId": form_str} + ): instance_res = self._current_response content = instance_res.content.strip() default_storage.save(instance_path, ContentFile(content)) @@ -232,12 +232,12 @@ def download_instances(self, form_id, cursor=0, num_entries=100): except ExpatError: continue - media_path = os.path.join(path, uuid.replace(':', '')) + media_path = os.path.join(path, uuid.replace(":", "")) self.download_media_files(instance_doc, media_path) self.logger.debug("Fetched %s %s submission" % (form_id, uuid)) - if xml_doc.getElementsByTagName('resumptionCursor'): - rs_node = xml_doc.getElementsByTagName('resumptionCursor')[0] + if xml_doc.getElementsByTagName("resumptionCursor"): + rs_node = xml_doc.getElementsByTagName("resumptionCursor")[0] cursor = rs_node.childNodes[0].nodeValue if self.resumption_cursor != cursor: @@ -258,19 +258,20 @@ def _upload_instance(self, xml_file, instance_dir_path, files): de_node = xml_doc.documentElement for node in de_node.firstChild.childNodes: xml.write(node.toxml()) - new_xml_file = ContentFile(xml.getvalue().encode('utf-8')) - new_xml_file.content_type = 'text/xml' + new_xml_file = ContentFile(xml.getvalue().encode("utf-8")) + new_xml_file.content_type = "text/xml" xml.close() attachments = [] - for attach in de_node.getElementsByTagName('mediaFile'): - filename_node = attach.getElementsByTagName('filename') + for attach in de_node.getElementsByTagName("mediaFile"): + filename_node = attach.getElementsByTagName("filename") filename = filename_node[0].childNodes[0].nodeValue if filename in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, filename)) + os.path.join(instance_dir_path, filename) + ) mimetype, encoding = mimetypes.guess_type(file_obj.name) - media_obj = django_file(file_obj, 'media_files[]', mimetype) + media_obj = django_file(file_obj, "media_files[]", mimetype) attachments.append(media_obj) create_instance(self.user.username, new_xml_file, attachments) @@ -284,9 +285,10 @@ def _upload_instances(self, path): i_dirs, files = default_storage.listdir(instance_dir_path) xml_file = None - if 'submission.xml' in files: + if "submission.xml" in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, 'submission.xml')) + os.path.join(instance_dir_path, "submission.xml") + ) xml_file = file_obj if xml_file: @@ -295,9 +297,12 @@ def _upload_instances(self, path): except ExpatError: continue except Exception as e: - logging.exception(_( - u'Ignoring exception, processing XML submission ' - 'raised exception: %s' % str(e))) + logging.exception( + _( + "Ignoring exception, processing XML submission " + "raised exception: %s" % str(e) + ) + ) else: instances_count += 1 @@ -308,7 +313,7 @@ def push(self): for form_dir in dirs: dir_path = os.path.join(self.forms_path, form_dir) form_dirs, form_files = default_storage.listdir(dir_path) - form_xml = '%s.xml' % form_dir + form_xml = "%s.xml" % form_dir if form_xml in form_files: form_xml_path = os.path.join(dir_path, form_xml) x = self._upload_xform(form_xml_path, form_xml) @@ -316,8 +321,7 @@ def push(self): self.logger.error("Failed to publish %s" % form_dir) else: self.logger.debug("Successfully published %s" % form_dir) - if 'instances' in form_dirs: + if "instances" in form_dirs: self.logger.debug("Uploading instances") - c = self._upload_instances(os.path.join(dir_path, 'instances')) - self.logger.debug("Published %d instances for %s" % - (c, form_dir)) + c = self._upload_instances(os.path.join(dir_path, "instances")) + self.logger.debug("Published %d instances for %s" % (c, form_dir)) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index f25a7cec52..db8fd9a805 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -25,7 +25,7 @@ from django.core.files.storage import default_storage from django.utils import timezone from django.utils.translation import ugettext as _ -from future.utils import iteritems +from six import iteritems from multidb.pinning import use_master from onadata.apps.logger.models import Instance, XForm @@ -33,24 +33,32 @@ from onadata.apps.messaging.serializers import send_message from onadata.celery import app from onadata.libs.utils import analytics -from onadata.libs.utils.async_status import (FAILED, async_status, - celery_state_to_status) -from onadata.libs.utils.common_tags import (MULTIPLE_SELECT_TYPE, EXCEL_TRUE, - XLS_DATE_FIELDS, - XLS_DATETIME_FIELDS, UUID, NA_REP, - INSTANCE_CREATE_EVENT, - INSTANCE_UPDATE_EVENT, - IMPORTED_VIA_CSV_BY) +from onadata.libs.utils.async_status import FAILED, async_status, celery_state_to_status +from onadata.libs.utils.common_tags import ( + MULTIPLE_SELECT_TYPE, + EXCEL_TRUE, + XLS_DATE_FIELDS, + XLS_DATETIME_FIELDS, + UUID, + NA_REP, + INSTANCE_CREATE_EVENT, + INSTANCE_UPDATE_EVENT, + IMPORTED_VIA_CSV_BY, +) from onadata.libs.utils.common_tools import report_exception from onadata.libs.utils.dict_tools import csv_dict_to_nested_dict -from onadata.libs.utils.logger_tools import (OpenRosaResponse, dict2xml, - safe_create_instance) +from onadata.libs.utils.logger_tools import ( + OpenRosaResponse, + dict2xml, + safe_create_instance, +) from onadata.libs.serializers.metadata_serializer import MetaDataSerializer DEFAULT_UPDATE_BATCH = 100 -PROGRESS_BATCH_UPDATE = getattr(settings, 'EXPORT_TASK_PROGRESS_UPDATE_BATCH', - DEFAULT_UPDATE_BATCH) -IGNORED_COLUMNS = ['formhub/uuid', 'meta/instanceID'] +PROGRESS_BATCH_UPDATE = getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH +) +IGNORED_COLUMNS = ["formhub/uuid", "meta/instanceID"] def get_submission_meta_dict(xform, instance_id): @@ -66,18 +74,17 @@ def get_submission_meta_dict(xform, instance_id): :return: The metadata dict :rtype: dict """ - uuid_arg = instance_id or 'uuid:{}'.format(uuid.uuid4()) - meta = {'instanceID': uuid_arg} + uuid_arg = instance_id or "uuid:{}".format(uuid.uuid4()) + meta = {"instanceID": uuid_arg} update = 0 - if instance_id and xform.instances.filter( - uuid=instance_id.replace('uuid:', '')).count() > 0: - uuid_arg = 'uuid:{}'.format(uuid.uuid4()) - meta.update({ - 'instanceID': uuid_arg, - 'deprecatedID': instance_id - }) + if ( + instance_id + and xform.instances.filter(uuid=instance_id.replace("uuid:", "")).count() > 0 + ): + uuid_arg = "uuid:{}".format(uuid.uuid4()) + meta.update({"instanceID": uuid_arg, "deprecatedID": instance_id}) update += 1 return [meta, update] @@ -94,16 +101,21 @@ def dict2xmlsubmission(submission_dict, xform, instance_id, submission_date): :rtype: string """ - return (u'' - '<{0} id="{1}" instanceID="uuid:{2}" submissionDate="{3}">{4}' - ''.format( - json.loads(xform.json).get('name', xform.id_string), - xform.id_string, instance_id, submission_date, - dict2xml(submission_dict).replace('\n', ''))).encode('utf-8') + return ( + '' + '<{0} id="{1}" instanceID="uuid:{2}" submissionDate="{3}">{4}' + "".format( + json.loads(xform.json).get("name", xform.id_string), + xform.id_string, + instance_id, + submission_date, + dict2xml(submission_dict).replace("\n", ""), + ) + ).encode("utf-8") def dict_merge(a, b): - """ Returns a merger of two dicts a and b + """Returns a merger of two dicts a and b credits: https://www.xormedia.com/recursively-merge-dictionaries-in-python @@ -124,7 +136,7 @@ def dict_merge(a, b): def dict_pathkeys_to_nested_dicts(dictionary): - """ Turns a flat dict to a nested dict + """Turns a flat dict to a nested dict Takes a dict with pathkeys or "slash-namespaced" keys and inflates them into nested dictionaries i.e:- @@ -136,11 +148,13 @@ def dict_pathkeys_to_nested_dicts(dictionary): """ data = dictionary.copy() for key in list(data): - if r'/' in key: + if r"/" in key: data = dict_merge( - functools.reduce(lambda v, k: {k: v}, - (key.split('/') + [data.pop(key)])[::-1]), - data) + functools.reduce( + lambda v, k: {k: v}, (key.split("/") + [data.pop(key)])[::-1] + ), + data, + ) return data @@ -155,7 +169,7 @@ def submit_csv_async(username, xform_id, file_path, overwrite=False): def failed_import(rollback_uuids, xform, exception, status_message): - """ Report a failed import. + """Report a failed import. :param rollback_uuids: The rollback UUIDs :param xform: The XForm that failed to import to :param exception: The exception object @@ -164,9 +178,10 @@ def failed_import(rollback_uuids, xform, exception, status_message): Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() xform.submission_count(True) report_exception( - 'CSV Import Failed : %d - %s - %s' % (xform.pk, xform.id_string, - xform.title), exception, - sys.exc_info()) + "CSV Import Failed : %d - %s - %s" % (xform.pk, xform.id_string, xform.title), + exception, + sys.exc_info(), + ) return async_status(FAILED, status_message) @@ -187,38 +202,46 @@ def validate_csv_file(csv_file, xform): # Validate csv_file is utf-8 encoded or unicode if isinstance(csv_file, str): csv_file = BytesIO(csv_file) - elif csv_file is None or not hasattr(csv_file, 'read'): + elif csv_file is None or not hasattr(csv_file, "read"): return { - 'error_msg': ( - u'Invalid param type for csv_file`.' - 'Expected utf-8 encoded file or unicode' - ' string got {} instead'.format(type(csv_file).__name__) - ), - 'valid': False} + "error_msg": ( + "Invalid param type for csv_file`." + "Expected utf-8 encoded file or unicode" + " string got {} instead".format(type(csv_file).__name__) + ), + "valid": False, + } # Ensure stream position is at the start of the file csv_file.seek(0) # Retrieve CSV Headers from the CSV File - csv_headers = ucsv.DictReader(csv_file, encoding='utf-8-sig').fieldnames + csv_headers = ucsv.DictReader(csv_file, encoding="utf-8-sig").fieldnames # Make sure CSV headers have no spaces # because these are converted to XLSForm names # which cannot have spaces - if any(' ' in header for header in csv_headers): + if any(" " in header for header in csv_headers): return { - 'error_msg': 'CSV file fieldnames should not contain spaces', - 'valid': False} + "error_msg": "CSV file fieldnames should not contain spaces", + "valid": False, + } # Get headers from stored data dictionary xform_headers = xform.get_headers() # Identify any missing columns between XForm and # imported CSV ignoring repeat and metadata columns - missing_col = sorted([ - col for col in set(xform_headers).difference(csv_headers) - if col.find('[') == -1 and not col.startswith('_') - and col not in IGNORED_COLUMNS and '/_' not in col]) + missing_col = sorted( + [ + col + for col in set(xform_headers).difference(csv_headers) + if col.find("[") == -1 + and not col.startswith("_") + and col not in IGNORED_COLUMNS + and "/_" not in col + ] + ) mutliple_select_col = [] @@ -226,32 +249,35 @@ def validate_csv_file(csv_file, xform): # the missing_col list for col in csv_headers: survey_element = xform.get_survey_element(col) - if survey_element and \ - survey_element.get('type') == MULTIPLE_SELECT_TYPE: + if survey_element and survey_element.get("type") == MULTIPLE_SELECT_TYPE: # remove from the missing list missing_col = [x for x in missing_col if not x.startswith(col)] mutliple_select_col.append(col) if missing_col: return { - 'error_msg': ( - u"Sorry uploaded file does not match the form. " - u"The file is missing the column(s): " - u"{0}.".format(', '.join(missing_col)) + "error_msg": ( + "Sorry uploaded file does not match the form. " + "The file is missing the column(s): " + "{0}.".format(", ".join(missing_col)) ), - 'valid': False} + "valid": False, + } # Identify any additional columns between XForm and # imported CSV ignoring repeat and multiple_select columns additional_col = [ - col for col in set(csv_headers).difference(xform_headers) - if col.find('[') == -1 and col not in mutliple_select_col] + col + for col in set(csv_headers).difference(xform_headers) + if col.find("[") == -1 and col not in mutliple_select_col + ] - return {'valid': True, 'additional_col': additional_col} + return {"valid": True, "additional_col": additional_col} def flatten_split_select_multiples( - row: Dict[str, Any], select_multiples: List[str]) -> dict: + row: Dict[str, Any], select_multiples: List[str] +) -> dict: """ Flattens a select_multiple question that was previously split into different choice columns into one column @@ -259,14 +285,13 @@ def flatten_split_select_multiples( for key, value in row.items(): if key in select_multiples and isinstance(value, dict): picked_choices = [ - k for k, v in value.items() - if v in ['1', 'TRUE'] or v == k] - new_value = ' '.join(picked_choices) + k for k, v in value.items() if v in ["1", "TRUE"] or v == k + ] + new_value = " ".join(picked_choices) row.update({key: new_value}) elif isinstance(value, dict): # Handle cases where select_multiples are within a group - new_value = flatten_split_select_multiples( - value, select_multiples) + new_value = flatten_split_select_multiples(value, select_multiples) row.update({key: new_value}) return row @@ -287,25 +312,22 @@ def submit_csv(username, xform, csv_file, overwrite=False): """ csv_file_validation_summary = validate_csv_file(csv_file, xform) - if csv_file_validation_summary.get('valid'): - additional_col = csv_file_validation_summary.get('additional_col') + if csv_file_validation_summary.get("valid"): + additional_col = csv_file_validation_summary.get("additional_col") else: - return async_status( - FAILED, - csv_file_validation_summary.get('error_msg') - ) + return async_status(FAILED, csv_file_validation_summary.get("error_msg")) num_rows = sum(1 for row in csv_file) - 1 # Change stream position to start of file csv_file.seek(0) - csv_reader = ucsv.DictReader(csv_file, encoding='utf-8-sig') + csv_reader = ucsv.DictReader(csv_file, encoding="utf-8-sig") xform_json = json.loads(xform.json) select_multiples = [ - qstn.name for qstn in - xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE)] - ona_uuid = {'formhub': {'uuid': xform.uuid}} + qstn.name for qstn in xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE) + ] + ona_uuid = {"formhub": {"uuid": xform.uuid}} additions = duplicates = inserts = 0 rollback_uuids = [] errors = {} @@ -313,25 +335,27 @@ def submit_csv(username, xform, csv_file, overwrite=False): # Retrieve the columns we should validate values for # Currently validating date, datetime, integer and decimal columns col_to_validate = { - 'date': (get_columns_by_type(XLS_DATE_FIELDS, xform_json), parse), - 'datetime': ( - get_columns_by_type(XLS_DATETIME_FIELDS, xform_json), parse), - 'integer': (get_columns_by_type(['integer'], xform_json), int), - 'decimal': (get_columns_by_type(['decimal'], xform_json), float) + "date": (get_columns_by_type(XLS_DATE_FIELDS, xform_json), parse), + "datetime": (get_columns_by_type(XLS_DATETIME_FIELDS, xform_json), parse), + "integer": (get_columns_by_type(["integer"], xform_json), int), + "decimal": (get_columns_by_type(["decimal"], xform_json), float), } if overwrite: - instance_ids = [i['id'] for i in xform.instances.values('id')] - xform.instances.filter(deleted_at__isnull=True)\ - .update(deleted_at=timezone.now(), - deleted_by=User.objects.get(username=username)) + instance_ids = [i["id"] for i in xform.instances.values("id")] + xform.instances.filter(deleted_at__isnull=True).update( + deleted_at=timezone.now(), deleted_by=User.objects.get(username=username) + ) # updates the form count xform.submission_count(True) # send message send_message( - instance_id=instance_ids, target_id=xform.id, - target_type=XFORM, user=User.objects.get(username=username), - message_verb=SUBMISSION_DELETED) + instance_id=instance_ids, + target_id=xform.id, + target_type=XFORM, + user=User.objects.get(username=username), + message_verb=SUBMISSION_DELETED, + ) try: for row_no, row in enumerate(csv_reader): @@ -340,7 +364,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): del row[index] # Remove 'n/a' and '' values from csv - row = {k: v for (k, v) in row.items() if v not in [NA_REP, '']} + row = {k: v for (k, v) in row.items() if v not in [NA_REP, ""]} row, error = validate_row(row, col_to_validate) @@ -355,73 +379,88 @@ def submit_csv(username, xform, csv_file, overwrite=False): for key in list(row): # Collect row location data into separate location_data # dict - if key.endswith(('.latitude', '.longitude', '.altitude', - '.precision')): - location_key, location_prop = key.rsplit(u'.', 1) - location_data.setdefault(location_key, {}).update({ - location_prop: - row.get(key, '0') - }) + if key.endswith( + (".latitude", ".longitude", ".altitude", ".precision") + ): + location_key, location_prop = key.rsplit(".", 1) + location_data.setdefault(location_key, {}).update( + {location_prop: row.get(key, "0")} + ) # collect all location K-V pairs into single geopoint field(s) # in location_data dict for location_key in list(location_data): - location_data.update({ - location_key: - (u'%(latitude)s %(longitude)s ' - '%(altitude)s %(precision)s') % defaultdict( - lambda: '', location_data.get(location_key)) - }) + location_data.update( + { + location_key: ( + "%(latitude)s %(longitude)s " + "%(altitude)s %(precision)s" + ) + % defaultdict(lambda: "", location_data.get(location_key)) + } + ) nested_dict = csv_dict_to_nested_dict( - row, select_multiples=select_multiples) + row, select_multiples=select_multiples + ) row = flatten_split_select_multiples( - nested_dict, select_multiples=select_multiples) + nested_dict, select_multiples=select_multiples + ) location_data = csv_dict_to_nested_dict(location_data) # Merge location_data into the Row data row = dict_merge(row, location_data) submission_time = datetime.utcnow().isoformat() - row_uuid = row.get('meta/instanceID') or 'uuid:{}'.format( - row.get(UUID)) if row.get(UUID) and not overwrite else None - submitted_by = row.get('_submitted_by') - submission_date = row.get('_submission_time', submission_time) + row_uuid = ( + row.get("meta/instanceID") or "uuid:{}".format(row.get(UUID)) + if row.get(UUID) and not overwrite + else None + ) + submitted_by = row.get("_submitted_by") + submission_date = row.get("_submission_time", submission_time) for key in list(row): # remove metadata (keys starting with '_') - if key.startswith('_'): + if key.startswith("_"): del row[key] # Inject our forms uuid into the submission row.update(ona_uuid) - old_meta = row.get('meta', {}) + old_meta = row.get("meta", {}) new_meta, update = get_submission_meta_dict(xform, row_uuid) inserts += update old_meta.update(new_meta) - row.update({'meta': old_meta}) + row.update({"meta": old_meta}) - row_uuid = row.get('meta').get('instanceID') - rollback_uuids.append(row_uuid.replace('uuid:', '')) + row_uuid = row.get("meta").get("instanceID") + rollback_uuids.append(row_uuid.replace("uuid:", "")) try: xml_file = BytesIO( - dict2xmlsubmission( - row, xform, row_uuid, submission_date)) + dict2xmlsubmission(row, xform, row_uuid, submission_date) + ) try: error, instance = safe_create_instance( - username, xml_file, [], xform.uuid, None, - instance_status='imported_via_csv' + username, + xml_file, + [], + xform.uuid, + None, + instance_status="imported_via_csv", ) except ValueError as e: error = e if error: - if not (isinstance(error, OpenRosaResponse) - and error.status_code == 202): + if not ( + isinstance(error, OpenRosaResponse) + and error.status_code == 202 + ): Instance.objects.filter( - uuid__in=rollback_uuids, xform=xform).delete() + uuid__in=rollback_uuids, xform=xform + ).delete() return async_status(FAILED, text(error)) else: duplicates += 1 @@ -431,28 +470,37 @@ def submit_csv(username, xform, csv_file, overwrite=False): if additions % PROGRESS_BATCH_UPDATE == 0: try: current_task.update_state( - state='PROGRESS', + state="PROGRESS", meta={ - 'progress': additions, - 'total': num_rows, - 'info': additional_col - }) + "progress": additions, + "total": num_rows, + "info": additional_col, + }, + ) except Exception: logging.exception( - _(u'Could not update state of ' - 'import CSV batch process.')) + _( + "Could not update state of " + "import CSV batch process." + ) + ) finally: xform.submission_count(True) - users = User.objects.filter( - username=submitted_by) if submitted_by else [] + users = ( + User.objects.filter(username=submitted_by) + if submitted_by + else [] + ) # Store user who imported data in metadata - serializer = MetaDataSerializer(data={ - "instance": instance.id, - "data_type": IMPORTED_VIA_CSV_BY, - "data_value": username - }) + serializer = MetaDataSerializer( + data={ + "instance": instance.id, + "data_type": IMPORTED_VIA_CSV_BY, + "data_value": username, + } + ) if serializer.is_valid(): serializer.save() @@ -463,19 +511,16 @@ def submit_csv(username, xform, csv_file, overwrite=False): except Exception as e: return failed_import(rollback_uuids, xform, e, text(e)) except UnicodeDecodeError as e: - return failed_import(rollback_uuids, xform, e, - 'CSV file must be utf-8 encoded') + return failed_import(rollback_uuids, xform, e, "CSV file must be utf-8 encoded") if errors: # Rollback all created instances if an error occurred during # validation - Instance.objects.filter( - uuid__in=rollback_uuids, xform=xform).delete() + Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() xform.submission_count(True) return async_status( FAILED, - u'Invalid CSV data imported in row(s): {}'.format( - errors) if errors else '' + "Invalid CSV data imported in row(s): {}".format(errors) if errors else "", ) else: xform.submission_count(True) @@ -483,58 +528,57 @@ def submit_csv(username, xform, csv_file, overwrite=False): event_by = User.objects.get(username=username) event_name = None tracking_properties = { - 'xform_id': xform.pk, - 'project_id': xform.project.pk, - 'submitted_by': event_by, - 'label': f'csv-import-for-form-{xform.pk}', - 'from': 'CSV Import', + "xform_id": xform.pk, + "project_id": xform.project.pk, + "submitted_by": event_by, + "label": f"csv-import-for-form-{xform.pk}", + "from": "CSV Import", } if added_submissions > 0: - tracking_properties['value'] = added_submissions + tracking_properties["value"] = added_submissions event_name = INSTANCE_CREATE_EVENT - analytics.track( - event_by, event_name, properties=tracking_properties) + analytics.track(event_by, event_name, properties=tracking_properties) if inserts > 0: - tracking_properties['value'] = inserts + tracking_properties["value"] = inserts event_name = INSTANCE_UPDATE_EVENT - analytics.track( - event_by, event_name, properties=tracking_properties) + analytics.track(event_by, event_name, properties=tracking_properties) return { - 'additions': added_submissions, - 'duplicates': duplicates, - 'updates': inserts, - 'info': "Additional column(s) excluded from the upload: '{0}'." - .format(', '.join(list(additional_col)))} + "additions": added_submissions, + "duplicates": duplicates, + "updates": inserts, + "info": "Additional column(s) excluded from the upload: '{0}'.".format( + ", ".join(list(additional_col)) + ), + } def get_async_csv_submission_status(job_uuid): - """ Gets CSV Submision progress or result + """Gets CSV Submision progress or result Can be used to pol long running submissions :param str job_uuid: The submission job uuid returned by _submit_csv.delay :return: Dict with import progress info (insertions & total) :rtype: Dict """ if not job_uuid: - return async_status(FAILED, u'Empty job uuid') + return async_status(FAILED, "Empty job uuid") job = AsyncResult(job_uuid) try: # result = (job.result or job.state) - if job.state not in ['SUCCESS', 'FAILURE']: + if job.state not in ["SUCCESS", "FAILURE"]: response = async_status(celery_state_to_status(job.state)) if isinstance(job.info, dict): response.update(job.info) return response - if job.state == 'FAILURE': - return async_status( - celery_state_to_status(job.state), text(job.result)) + if job.state == "FAILURE": + return async_status(celery_state_to_status(job.state), text(job.result)) except BacklogLimitExceeded: - return async_status(celery_state_to_status('PENDING')) + return async_status(celery_state_to_status("PENDING")) return job.get() @@ -567,8 +611,10 @@ def submission_xls_to_csv(xls_file): # a null and thus XLS Dates (floats) or XLS Booleans # will not be properly converted in the next steps, # therefore we find the first non-empty row. - while first_sheet.cell_type(row, index) == xlrd.XL_CELL_EMPTY \ - and row < first_sheet.nrows - 1: + while ( + first_sheet.cell_type(row, index) == xlrd.XL_CELL_EMPTY + and row < first_sheet.nrows - 1 + ): row += 1 if first_sheet.cell_type(row, index) == xlrd.XL_CELL_DATE: @@ -583,16 +629,14 @@ def submission_xls_to_csv(xls_file): for date_column in date_columns: try: row_values[date_column] = xlrd.xldate_as_datetime( - row_values[date_column], - xl_workbook.datemode).isoformat() + row_values[date_column], xl_workbook.datemode + ).isoformat() except (ValueError, TypeError): - row_values[date_column] = first_sheet.cell_value( - row, date_column) + row_values[date_column] = first_sheet.cell_value(row, date_column) # convert excel boolean to true/false for boolean_column in boolean_columns: - row_values[boolean_column] = bool( - row_values[boolean_column] == EXCEL_TRUE) + row_values[boolean_column] = bool(row_values[boolean_column] == EXCEL_TRUE) csv_writer.writerow(row_values) @@ -616,16 +660,12 @@ def _column_by_type(item_list, prefix=""): found = [] for item in item_list: if item["type"] in ["group", "repeat"]: - prefix = "/".join( - [prefix, item["name"]] - ) if prefix else item["name"] + prefix = "/".join([prefix, item["name"]]) if prefix else item["name"] found.extend(_column_by_type(item["children"], prefix)) prefix = "" # Reset prefix to blank else: if item["type"] in type_list: - name = "%s/%s" % ( - prefix, item["name"] - ) if prefix else item["name"] + name = "%s/%s" % (prefix, item["name"]) if prefix else item["name"] found.append(name) return found @@ -654,7 +694,7 @@ def validate_row(row, columns): valid, data = validate_column(row, column, constraint_check) if valid: - if datatype in ['date', 'datetime']: + if datatype in ["date", "datetime"]: for key in data: # ODK XForms accept date and datetime values formatted in # accordance to the XML 1.0 spec only deviating when it @@ -662,18 +702,17 @@ def validate_row(row, columns): # datetime values. This specification matches ISO 8601 value = data.get(key).isoformat() - if datatype == 'date' and value.endswith('T00:00:00'): + if datatype == "date" and value.endswith("T00:00:00"): # Remove the Time string section from dates # to follow along with the date datetype specification # in the ODK XForms Spec - value = value.replace('T00:00:00', '') + value = value.replace("T00:00:00", "") data.update({key: value}) row.update(data) else: - errors.append('Unknown {} format(s): {}'.format( - datatype, ', '.join(data))) + errors.append("Unknown {} format(s): {}".format(datatype, ", ".join(data))) return (row, errors) @@ -696,7 +735,7 @@ def validate_column(row, columns, constraint_check): validated_data = {} for key in columns: - value = row.get(key, '') + value = row.get(key, "") if value: try: @@ -706,5 +745,4 @@ def validate_column(row, columns, constraint_check): else: validated_data[key] = value - return (False, invalid_data) if invalid_data \ - else (True, validated_data) + return (False, invalid_data) if invalid_data else (True, validated_data) diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 89e1d19fa4..0d556d8efe 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,5 +1,5 @@ from functools import wraps -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME from django.utils.decorators import available_attrs @@ -21,20 +21,22 @@ def is_owner(view_func): def _wrapped_view(request, *args, **kwargs): # assume username is first arg if request.user.is_authenticated: - if request.user.username == kwargs['username']: + if request.user.username == kwargs["username"]: return view_func(request, *args, **kwargs) protocol = "https" if request.is_secure() else "http" - return HttpResponseRedirect("%s://%s" % (protocol, - request.get_host())) + return HttpResponseRedirect("%s://%s" % (protocol, request.get_host())) path = request.build_absolute_uri() login_url = request.build_absolute_uri(settings.LOGIN_URL) # If the login url is the same scheme and net location then just # use the path as the "next" url. login_scheme, login_netloc = urlparse(login_url)[:2] current_scheme, current_netloc = urlparse(path)[:2] - if ((not login_scheme or login_scheme == current_scheme) and - (not login_netloc or login_netloc == current_netloc)): + if (not login_scheme or login_scheme == current_scheme) and ( + not login_netloc or login_netloc == current_netloc + ): path = request.get_full_path() from django.contrib.auth.views import redirect_to_login + return redirect_to_login(path, None, REDIRECT_FIELD_NAME) + return _wrapped_view diff --git a/onadata/libs/utils/email.py b/onadata/libs/utils/email.py index d67a9d4731..9b11a5ada7 100644 --- a/onadata/libs/utils/email.py +++ b/onadata/libs/utils/email.py @@ -1,84 +1,68 @@ from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from rest_framework.reverse import reverse def get_verification_url(redirect_url, request, verification_key): verification_url = getattr(settings, "VERIFICATION_URL", None) - url = verification_url or reverse( - 'userprofile-verify-email', request=request - ) - query_params_dict = {'verification_key': verification_key} - redirect_url and query_params_dict.update({ - 'redirect_url': redirect_url - }) + url = verification_url or reverse("userprofile-verify-email", request=request) + query_params_dict = {"verification_key": verification_key} + redirect_url and query_params_dict.update({"redirect_url": redirect_url}) query_params_string = urlencode(query_params_dict) - verification_url = '{}?{}'.format(url, query_params_string) + verification_url = "{}?{}".format(url, query_params_string) return verification_url def get_verification_email_data(email, username, verification_url, request): - email_data = {'email': email} + email_data = {"email": email} ctx_dict = { - 'username': username, - 'expiration_days': getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), - 'verification_url': verification_url + "username": username, + "expiration_days": getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), + "verification_url": verification_url, } key_template_path_dict = { - 'subject': 'registration/verification_email_subject.txt', - 'message_txt': 'registration/verification_email.txt' + "subject": "registration/verification_email_subject.txt", + "message_txt": "registration/verification_email.txt", } for key, template_path in key_template_path_dict.items(): - email_data.update({ - key: render_to_string( - template_path, - ctx_dict, - request=request - ) - }) + email_data.update( + {key: render_to_string(template_path, ctx_dict, request=request)} + ) return email_data def get_account_lockout_email_data(username, ip, end=False): """Generates both the email upon start and end of account lockout""" - message_path = 'account_lockout/lockout_start.txt' - subject_path = 'account_lockout/lockout_email_subject.txt' + message_path = "account_lockout/lockout_start.txt" + subject_path = "account_lockout/lockout_email_subject.txt" if end: - message_path = 'account_lockout/lockout_end.txt' + message_path = "account_lockout/lockout_end.txt" ctx_dict = { - 'username': username, - 'remote_ip': ip, - 'lockout_time': getattr(settings, 'LOCKOUT_TIME', 1800) / 60, - 'support_email': getattr( - settings, 'SUPPORT_EMAIL', 'support@example.com') + "username": username, + "remote_ip": ip, + "lockout_time": getattr(settings, "LOCKOUT_TIME", 1800) / 60, + "support_email": getattr(settings, "SUPPORT_EMAIL", "support@example.com"), } email_data = { - 'subject': render_to_string(subject_path), - 'message_txt': render_to_string(message_path, ctx_dict) + "subject": render_to_string(subject_path), + "message_txt": render_to_string(message_path, ctx_dict), } return email_data def send_generic_email(email, message_txt, subject): - if any(a in [None, ''] for a in [email, message_txt, subject]): - raise ValueError( - "email, message_txt amd subject arguments are ALL required." - ) + if any(a in [None, ""] for a in [email, message_txt, subject]): + raise ValueError("email, message_txt amd subject arguments are ALL required.") from_email = settings.DEFAULT_FROM_EMAIL - email_message = EmailMultiAlternatives( - subject, - message_txt, - from_email, - [email] - ) + email_message = EmailMultiAlternatives(subject, message_txt, from_email, [email]) email_message.send() diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 9d0249737c..018b7f74ec 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -18,7 +18,7 @@ from django.core.files.temp import NamedTemporaryFile from onadata.libs.utils.common_tools import str_to_bool from django.utils.translation import ugettext as _ -from future.utils import iteritems +from six import iteritems from openpyxl.utils.datetime import to_excel from openpyxl.workbook import Workbook from pyxform.question import Question @@ -26,20 +26,44 @@ from savReaderWriter import SavWriter from onadata.apps.logger.models.osmdata import OsmData -from onadata.apps.logger.models.xform import (QUESTION_TYPES_TO_EXCLUDE, - _encode_for_mongo) +from onadata.apps.logger.models.xform import ( + QUESTION_TYPES_TO_EXCLUDE, + _encode_for_mongo, +) from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DELETEDAT, DURATION, GEOLOCATION, - ID, INDEX, MULTIPLE_SELECT_TYPE, SELECT_ONE, NOTES, PARENT_INDEX, - PARENT_TABLE_NAME, REPEAT_INDEX_TAGS, SAV_255_BYTES_TYPE, - SAV_NUMERIC_TYPE, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, UUID, - VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, SELECT_BIND_TYPE, - REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DELETEDAT, + DURATION, + GEOLOCATION, + ID, + INDEX, + MULTIPLE_SELECT_TYPE, + SELECT_ONE, + NOTES, + PARENT_INDEX, + PARENT_TABLE_NAME, + REPEAT_INDEX_TAGS, + SAV_255_BYTES_TYPE, + SAV_NUMERIC_TYPE, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + UUID, + VERSION, + XFORM_ID_STRING, + REVIEW_STATUS, + REVIEW_COMMENT, + SELECT_BIND_TYPE, + REVIEW_DATE, +) from onadata.libs.utils.mongo import _decode_from_mongo, _is_invalid_for_mongo + # the bind type of select multiples that we use to compare -GEOPOINT_BIND_TYPE = 'geopoint' -OSM_BIND_TYPE = 'osm' +GEOPOINT_BIND_TYPE = "geopoint" +OSM_BIND_TYPE = "osm" DEFAULT_UPDATE_BATCH = 100 YES = 1 @@ -56,14 +80,15 @@ def current_site_url(path): :return: complete url """ from django.contrib.sites.models import Site + current_site = Site.objects.get_current() - protocol = getattr(settings, 'ONA_SITE_PROTOCOL', 'http') - port = getattr(settings, 'ONA_SITE_PORT', '') - url = '%s://%s' % (protocol, current_site.domain) + protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") + port = getattr(settings, "ONA_SITE_PORT", "") + url = "%s://%s" % (protocol, current_site.domain) if port: - url += ':%s' % port + url += ":%s" % port if path: - url += '%s' % path + url += "%s" % path return url @@ -74,8 +99,11 @@ def get_choice_label(label, data_dictionary, language=None): """ if isinstance(label, dict): languages = [i for i in label.keys()] - _language = language if language in languages else \ - data_dictionary.get_language(languages) + _language = ( + language + if language in languages + else data_dictionary.get_language(languages) + ) return label[_language] @@ -87,12 +115,12 @@ def get_choice_label_value(key, value, data_dictionary, language=None): Return the label of a choice matching the value if the key xpath is a SELECT_ONE otherwise it returns the value unchanged. """ + def _get_choice_label_value(lookup): _label = None for choice in data_dictionary.get_survey_element(key).children: if choice.name == lookup: - _label = get_choice_label(choice.label, data_dictionary, - language) + _label = get_choice_label(choice.label, data_dictionary, language) break return _label @@ -103,27 +131,34 @@ def _get_choice_label_value(lookup): if key in data_dictionary.get_select_multiple_xpaths(): answers = [] - for item in value.split(' '): + for item in value.split(" "): answer = _get_choice_label_value(item) answers.append(answer or item) if [_i for _i in answers if _i is not None]: - label = ' '.join(answers) + label = " ".join(answers) return label or value def get_value_or_attachment_uri( # pylint: disable=too-many-arguments - key, value, row, data_dictionary, media_xpaths, - attachment_list=None, show_choice_labels=False, language=None): + key, + value, + row, + data_dictionary, + media_xpaths, + attachment_list=None, + show_choice_labels=False, + language=None, +): """ - Gets either the attachment value or the attachment url - :param key: used to retrieve survey element - :param value: filename - :param row: current records row - :param data_dictionary: form structure - :param include_images: boolean value to either inlcude images or not - :param attachment_list: to be used incase row doesn't have ATTACHMENTS key - :return: value + Gets either the attachment value or the attachment url + :param key: used to retrieve survey element + :param value: filename + :param row: current records row + :param data_dictionary: form structure + :param include_images: boolean value to either inlcude images or not + :param attachment_list: to be used incase row doesn't have ATTACHMENTS key + :return: value """ if show_choice_labels: value = get_choice_label_value(key, value, data_dictionary, language) @@ -135,10 +170,10 @@ def get_value_or_attachment_uri( # pylint: disable=too-many-arguments attachments = [ a for a in row.get(ATTACHMENTS, attachment_list or []) - if a.get('name') == value + if a.get("name") == value ] if attachments: - value = current_site_url(attachments[0].get('download_url', '')) + value = current_site_url(attachments[0].get("download_url", "")) return value @@ -156,24 +191,24 @@ def encode_if_str(row, key, encode_dates=False, sav_writer=None): if sav_writer: if isinstance(val, datetime): if len(val.isoformat()): - strptime_fmt = '%Y-%m-%dT%H:%M:%S' + strptime_fmt = "%Y-%m-%dT%H:%M:%S" else: - strptime_fmt = '%Y-%m-%dT%H:%M:%S.%f%z' + strptime_fmt = "%Y-%m-%dT%H:%M:%S.%f%z" else: - strptime_fmt = '%Y-%m-%d' - return sav_writer.spssDateTime(val.isoformat().encode('utf-8'), - strptime_fmt) + strptime_fmt = "%Y-%m-%d" + return sav_writer.spssDateTime( + val.isoformat().encode("utf-8"), strptime_fmt + ) elif encode_dates: return val.isoformat() if sav_writer: - val = '' if val is None else val + val = "" if val is None else val return text(val) if IS_PY_3K and not isinstance(val, bool) else val return val -def dict_to_joined_export(data, index, indices, name, survey, row, - media_xpaths=[]): +def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths=[]): """ Converts a dict into one or more tabular datasets :param data: current record which can be changed or updated @@ -195,10 +230,13 @@ def dict_to_joined_export(data, index, indices, name, survey, row, indices[key] += 1 child_index = indices[key] new_output = dict_to_joined_export( - child, child_index, indices, key, survey, row, - media_xpaths) - d = {INDEX: child_index, PARENT_INDEX: index, - PARENT_TABLE_NAME: name} + child, child_index, indices, key, survey, row, media_xpaths + ) + d = { + INDEX: child_index, + PARENT_INDEX: index, + PARENT_TABLE_NAME: name, + } # iterate over keys within new_output and append to # main output for (out_key, out_val) in iteritems(new_output): @@ -213,16 +251,20 @@ def dict_to_joined_export(data, index, indices, name, survey, row, if name not in output: output[name] = {} if key in [TAGS]: - output[name][key] = ','.join(val) + output[name][key] = ",".join(val) elif key in [NOTES]: - note_list = [v if isinstance(v, text) - else v['note'] for v in val] - output[name][key] = '\r\n'.join(note_list) + note_list = [v if isinstance(v, text) else v["note"] for v in val] + output[name][key] = "\r\n".join(note_list) else: data_dictionary = get_data_dictionary_from_survey(survey) output[name][key] = get_value_or_attachment_uri( - key, val, data, data_dictionary, media_xpaths, - row and row.get(ATTACHMENTS)) + key, + val, + data, + data_dictionary, + media_xpaths, + row and row.get(ATTACHMENTS), + ) return output @@ -239,16 +281,22 @@ def is_all_numeric(items): for i in items: float(i) # if there is a zero padded number, it is not all numeric - if isinstance(i, text) and len(i) > 1 and \ - i[0] == '0' and i[1] != '.': + if isinstance(i, text) and len(i) > 1 and i[0] == "0" and i[1] != ".": return False return True except ValueError: return False # check for zero padded numbers to be treated as non numeric - return not (any([i.startswith('0') and len(i) > 1 and i.find('.') == -1 - for i in items if isinstance(i, text)])) + return not ( + any( + [ + i.startswith("0") and len(i) > 1 and i.find(".") == -1 + for i in items + if isinstance(i, text) + ] + ) + ) def track_task_progress(additions, total=None): @@ -261,19 +309,23 @@ def track_task_progress(additions, total=None): :return: """ try: - if additions % getattr(settings, 'EXPORT_TASK_PROGRESS_UPDATE_BATCH', - DEFAULT_UPDATE_BATCH) == 0: - meta = {'progress': additions} + if ( + additions + % getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH + ) + == 0 + ): + meta = {"progress": additions} if total: - meta.update({'total': total}) - current_task.update_state(state='PROGRESS', meta=meta) + meta.update({"total": total}) + current_task.update_state(state="PROGRESS", meta=meta) except Exception as e: - logging.exception( - _('Track task progress threw exception: %s' % text(e))) + logging.exception(_("Track task progress threw exception: %s" % text(e))) def string_to_date_with_xls_validation(date_str): - """ Try to convert a string to a date object. + """Try to convert a string to a date object. :param date_str: string to convert :returns: object if converted, otherwise date string @@ -282,7 +334,7 @@ def string_to_date_with_xls_validation(date_str): return date_str try: - date_obj = datetime.strptime(date_str, '%Y-%m-%d').date() + date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() to_excel(date_obj) except ValueError: return date_str @@ -291,7 +343,7 @@ def string_to_date_with_xls_validation(date_str): def decode_mongo_encoded_section_names(data): - """ Recursively decode mongo keys. + """Recursively decode mongo keys. :param data: A dictionary to decode. """ @@ -301,32 +353,49 @@ def decode_mongo_encoded_section_names(data): if isinstance(v, dict): new_v = decode_mongo_encoded_section_names(v) elif isinstance(v, list): - new_v = [decode_mongo_encoded_section_names(x) - if isinstance(x, dict) else x for x in v] + new_v = [ + decode_mongo_encoded_section_names(x) if isinstance(x, dict) else x + for x in v + ] results[_decode_from_mongo(k)] = new_v return results class ExportBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + ] # fields we export but are not within the form's structure EXTRA_FIELDS = [ - ID, UUID, SUBMISSION_TIME, INDEX, PARENT_TABLE_NAME, PARENT_INDEX, - TAGS, NOTES, VERSION, DURATION, - SUBMITTED_BY] + ID, + UUID, + SUBMISSION_TIME, + INDEX, + PARENT_TABLE_NAME, + PARENT_INDEX, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + ] SPLIT_SELECT_MULTIPLES = True BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False # column group delimiters get_value_or_attachment_uri - GROUP_DELIMITER_SLASH = '/' - GROUP_DELIMITER_DOT = '.' + GROUP_DELIMITER_SLASH = "/" + GROUP_DELIMITER_DOT = "." GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] # index tags - REPEAT_INDEX_TAGS = ('[', ']') + REPEAT_INDEX_TAGS = ("[", "]") INCLUDE_LABELS = False INCLUDE_LABELS_ONLY = False @@ -336,12 +405,12 @@ class ExportBuilder(object): SHOW_CHOICE_LABELS = False INCLUDE_REVIEWS = False - TYPES_TO_CONVERT = ['int', 'decimal', 'date'] # , 'dateTime'] + TYPES_TO_CONVERT = ["int", "decimal", "date"] # , 'dateTime'] CONVERT_FUNCS = { - 'int': int, - 'decimal': float, - 'date': string_to_date_with_xls_validation, - 'dateTime': lambda x: datetime.strptime(x[:19], '%Y-%m-%dT%H:%M:%S') + "int": int, + "decimal": float, + "date": string_to_date_with_xls_validation, + "dateTime": lambda x: datetime.strptime(x[:19], "%Y-%m-%dT%H:%M:%S"), } TRUNCATE_GROUP_TITLE = False @@ -351,13 +420,17 @@ class ExportBuilder(object): language = None def __init__(self): - self.extra_columns = ( - self.EXTRA_FIELDS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.EXTRA_FIELDS + getattr(settings, "EXTRA_COLUMNS", []) self.osm_columns = [] @classmethod - def format_field_title(cls, abbreviated_xpath, field_delimiter, - data_dictionary, remove_group_name=False): + def format_field_title( + cls, + abbreviated_xpath, + field_delimiter, + data_dictionary, + remove_group_name=False, + ): title = abbreviated_xpath # Check if to truncate the group name prefix if remove_group_name: @@ -365,13 +438,13 @@ def format_field_title(cls, abbreviated_xpath, field_delimiter, # incase abbreviated_xpath is a choices xpath if elem is None: pass - elif elem.type == '': - title = '/'.join([elem.parent.name, elem.name]) + elif elem.type == "": + title = "/".join([elem.parent.name, elem.name]) else: title = elem.name - if field_delimiter != '/': - title = field_delimiter.join(title.split('/')) + if field_delimiter != "/": + title = field_delimiter.join(title.split("/")) return title @@ -382,170 +455,232 @@ def get_choice_label_from_dict(self, label): return label - def _get_select_mulitples_choices(self, child, dd, field_delimiter, - remove_group_name): + def _get_select_mulitples_choices( + self, child, dd, field_delimiter, remove_group_name + ): def get_choice_dict(xpath, label): title = ExportBuilder.format_field_title( xpath, field_delimiter, dd, remove_group_name ) return { - 'label': field_delimiter.join([child.name, label or title]), - '_label': label or title, - '_label_xpath': field_delimiter.join([child.name, - label or title]), - 'title': title, - 'xpath': xpath, - 'type': 'string' + "label": field_delimiter.join([child.name, label or title]), + "_label": label or title, + "_label_xpath": field_delimiter.join([child.name, label or title]), + "title": title, + "xpath": xpath, + "type": "string", } choices = [] is_choice_randomized = str_to_bool( - child.parameters and child.parameters.get('randomize')) - if ((not child.children and child.choice_filter) - or is_choice_randomized) and child.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(child.itemset) - choices = [get_choice_dict( - '/'.join([child.get_abbreviated_xpath(), i['name']]), - self.get_choice_label_from_dict(i['label']) - ) for i in itemset] if itemset else choices + child.parameters and child.parameters.get("randomize") + ) + if ( + (not child.children and child.choice_filter) or is_choice_randomized + ) and child.itemset: + itemset = dd.survey.to_json_dict()["choices"].get(child.itemset) + choices = ( + [ + get_choice_dict( + "/".join([child.get_abbreviated_xpath(), i["name"]]), + self.get_choice_label_from_dict(i["label"]), + ) + for i in itemset + ] + if itemset + else choices + ) else: - choices = [get_choice_dict( - c.get_abbreviated_xpath(), - get_choice_label(c.label, dd, language=self.language)) - for c in child.children] + choices = [ + get_choice_dict( + c.get_abbreviated_xpath(), + get_choice_label(c.label, dd, language=self.language), + ) + for c in child.children + ] return choices def set_survey(self, survey, xform=None, include_reviews=False): if self.INCLUDE_REVIEWS or include_reviews: self.EXTRA_FIELDS = self.EXTRA_FIELDS + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] self.__init__() dd = get_data_dictionary_from_survey(survey) def build_sections( - current_section, survey_element, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, select_ones, - field_delimiter='/', remove_group_name=False, language=None): + current_section, + survey_element, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter="/", + remove_group_name=False, + language=None, + ): for child in survey_element.children: - current_section_name = current_section['name'] + current_section_name = current_section["name"] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { - 'name': child.get_abbreviated_xpath(), - 'elements': []} + "name": child.get_abbreviated_xpath(), + "elements": [], + } self.sections.append(section) build_sections( - section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) + section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) else: # its a group, recurs using the same section build_sections( - current_section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) - elif isinstance(child, Question) and \ - (child.bind.get('type') - not in QUESTION_TYPES_TO_EXCLUDE and - child.type not in QUESTION_TYPES_TO_EXCLUDE): + current_section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) + elif isinstance(child, Question) and ( + child.bind.get("type") not in QUESTION_TYPES_TO_EXCLUDE + and child.type not in QUESTION_TYPES_TO_EXCLUDE + ): # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( child.get_abbreviated_xpath(), - field_delimiter, dd, remove_group_name + field_delimiter, + dd, + remove_group_name, + ) + _label = ( + dd.get_label(child_xpath, elem=child, language=language) + or _title + ) + current_section["elements"].append( + { + "label": _label, + "title": _title, + "xpath": child_xpath, + "type": child.bind.get("type"), + } ) - _label = \ - dd.get_label( - child_xpath, - elem=child, - language=language) or _title - current_section['elements'].append({ - 'label': _label, - 'title': _title, - 'xpath': child_xpath, - 'type': child.bind.get('type') - }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( - {child_xpath: _encode_for_mongo(child_xpath)}) + {child_xpath: _encode_for_mongo(child_xpath)} + ) # if its a select multiple, make columns out of its choices - if child.bind.get('type') == SELECT_BIND_TYPE \ - and child.type == MULTIPLE_SELECT_TYPE: + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == MULTIPLE_SELECT_TYPE + ): choices = [] if self.SPLIT_SELECT_MULTIPLES: choices = self._get_select_mulitples_choices( child, dd, field_delimiter, remove_group_name ) for choice in choices: - if choice not in current_section['elements']: - current_section['elements'].append(choice) + if choice not in current_section["elements"]: + current_section["elements"].append(choice) # choices_xpaths = [c['xpath'] for c in choices] _append_xpaths_to_section( - current_section_name, select_multiples, - child.get_abbreviated_xpath(), choices) + current_section_name, + select_multiples, + child.get_abbreviated_xpath(), + choices, + ) # split gps fields within this section - if child.bind.get('type') == GEOPOINT_BIND_TYPE: + if child.bind.get("type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( - child.get_abbreviated_xpath()) + child.get_abbreviated_xpath() + ) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, field_delimiter, dd, remove_group_name + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "decimal", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'decimal' - }) _append_xpaths_to_section( - current_section_name, gps_fields, - child.get_abbreviated_xpath(), xpaths) + current_section_name, + gps_fields, + child.get_abbreviated_xpath(), + xpaths, + ) # get other osm fields - if child.get(u"type") == OSM_BIND_TYPE: + if child.get("type") == OSM_BIND_TYPE: xpaths = _get_osm_paths(child, xform) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, field_delimiter, dd, remove_group_name + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "osm", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'osm' - }) _append_xpaths_to_section( - current_section_name, osm_fields, - child.get_abbreviated_xpath(), xpaths) - if child.bind.get(u"type") == SELECT_BIND_TYPE \ - and child.type == SELECT_ONE: + current_section_name, + osm_fields, + child.get_abbreviated_xpath(), + xpaths, + ) + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == SELECT_ONE + ): _append_xpaths_to_section( - current_section_name, select_ones, - child.get_abbreviated_xpath(), []) + current_section_name, + select_ones, + child.get_abbreviated_xpath(), + [], + ) - def _append_xpaths_to_section(current_section_name, field_list, xpath, - xpaths): + def _append_xpaths_to_section(current_section_name, field_list, xpath, xpaths): if current_section_name not in field_list: field_list[current_section_name] = {} - field_list[ - current_section_name][xpath] = xpaths + field_list[current_section_name][xpath] = xpaths def _get_osm_paths(osm_field, xform): """ @@ -555,8 +690,8 @@ def _get_osm_paths(osm_field, xform): osm_columns = [] if osm_field and xform: osm_columns = OsmData.get_tag_keys( - xform, osm_field.get_abbreviated_xpath(), - include_prefix=True) + xform, osm_field.get_abbreviated_xpath(), include_prefix=True + ) return osm_columns self.dd = dd @@ -566,27 +701,40 @@ def _get_osm_paths(osm_field, xform): self.gps_fields = {} self.osm_fields = {} self.encoded_fields = {} - main_section = {'name': survey.name, 'elements': []} + main_section = {"name": survey.name, "elements": []} self.sections = [main_section] build_sections( - main_section, self.survey, self.sections, - self.select_multiples, self.gps_fields, self.osm_fields, - self.encoded_fields, self.select_ones, self.GROUP_DELIMITER, - self.TRUNCATE_GROUP_TITLE, language=self.language) + main_section, + self.survey, + self.sections, + self.select_multiples, + self.gps_fields, + self.osm_fields, + self.encoded_fields, + self.select_ones, + self.GROUP_DELIMITER, + self.TRUNCATE_GROUP_TITLE, + language=self.language, + ) def section_by_name(self, name): - matches = [s for s in self.sections if s['name'] == name] - assert(len(matches) == 1) + matches = [s for s in self.sections if s["name"] == name] + assert len(matches) == 1 return matches[0] # pylint: disable=too-many-arguments @classmethod - def split_select_multiples(cls, row, select_multiples, - select_values=False, - binary_select_multiples=False, - show_choice_labels=False, data_dictionary=None, - language=None): + def split_select_multiples( + cls, + row, + select_multiples, + select_values=False, + binary_select_multiples=False, + show_choice_labels=False, + data_dictionary=None, + language=None, + ): """ Split select multiple choices in a submission to individual columns. @@ -613,37 +761,69 @@ def split_select_multiples(cls, row, select_multiples, selections = [] if data: selections = [ - '{0}/{1}'.format( - xpath, selection) for selection in data.split()] + "{0}/{1}".format(xpath, selection) for selection in data.split() + ] if show_choice_labels and data_dictionary: row[xpath] = get_choice_label_value( - xpath, data, data_dictionary, language) + xpath, data, data_dictionary, language + ) if select_values: if show_choice_labels: - row.update(dict( - [(choice['label'], choice['_label'] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"], + choice["_label"] + if selections and choice["xpath"] in selections + else None, + ) + for choice in choices + ] + ) + ) else: - row.update(dict( - [(choice['xpath'], - data.split()[selections.index(choice['xpath'])] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["xpath"], + data.split()[selections.index(choice["xpath"])] + if selections and choice["xpath"] in selections + else None, + ) + for choice in choices + ] + ) + ) elif binary_select_multiples: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - YES if choice['xpath'] in selections else NO) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"] + if show_choice_labels + else choice["xpath"], + YES if choice["xpath"] in selections else NO, + ) + for choice in choices + ] + ) + ) else: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - choice['xpath'] in selections if selections else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"] + if show_choice_labels + else choice["xpath"], + choice["xpath"] in selections if selections else None, + ) + for choice in choices + ] + ) + ) return row @classmethod @@ -683,61 +863,70 @@ def pre_process_row(self, row, section): """ Split select multiples, gps and decode . and $ """ - section_name = section['name'] + section_name = section["name"] # first decode fields so that subsequent lookups # have decoded field names if section_name in self.encoded_fields: row = ExportBuilder.decode_mongo_encoded_fields( - row, self.encoded_fields[section_name]) + row, self.encoded_fields[section_name] + ) if section_name in self.select_multiples: select_multiples = self.select_multiples[section_name] if self.SPLIT_SELECT_MULTIPLES: row = ExportBuilder.split_select_multiples( - row, select_multiples, self.VALUE_SELECT_MULTIPLES, + row, + select_multiples, + self.VALUE_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, show_choice_labels=self.SHOW_CHOICE_LABELS, - data_dictionary=self.dd, language=self.language) + data_dictionary=self.dd, + language=self.language, + ) if not self.SPLIT_SELECT_MULTIPLES and self.SHOW_CHOICE_LABELS: for xpath in select_multiples: # get the data matching this xpath data = row.get(xpath) and text(row.get(xpath)) if data: row[xpath] = get_choice_label_value( - xpath, data, self.dd, self.language) + xpath, data, self.dd, self.language + ) if section_name in self.gps_fields: - row = ExportBuilder.split_gps_components( - row, self.gps_fields[section_name]) + row = ExportBuilder.split_gps_components(row, self.gps_fields[section_name]) if section_name in self.select_ones and self.SHOW_CHOICE_LABELS: for key in self.select_ones[section_name]: if key in row: - row[key] = get_choice_label_value(key, row[key], self.dd, - self.language) + row[key] = get_choice_label_value( + key, row[key], self.dd, self.language + ) # convert to native types - for elm in section['elements']: + for elm in section["elements"]: # only convert if its in our list and its not empty, just to # optimize - value = row.get(elm['xpath']) - if elm['type'] in ExportBuilder.TYPES_TO_CONVERT\ - and value is not None and value != '': - row[elm['xpath']] = ExportBuilder.convert_type( - value, elm['type']) + value = row.get(elm["xpath"]) + if ( + elm["type"] in ExportBuilder.TYPES_TO_CONVERT + and value is not None + and value != "" + ): + row[elm["xpath"]] = ExportBuilder.convert_type(value, elm["type"]) if SUBMISSION_TIME in row: row[SUBMISSION_TIME] = ExportBuilder.convert_type( - row[SUBMISSION_TIME], 'dateTime') + row[SUBMISSION_TIME], "dateTime" + ) # Map dynamic values for key, value in row.items(): if isinstance(value, str): - dynamic_val_regex = '\$\{\w+\}' # noqa + dynamic_val_regex = "\$\{\w+\}" # noqa # Find substrings that match ${`any_text`} result = re.findall(dynamic_val_regex, value) if result: for val in result: - val_key = val.replace('${', '').replace('}', '') + val_key = val.replace("${", "").replace("}", "") # Try retrieving value of ${`any_text`} from the # row data and replace the value if row.get(val_key): @@ -748,45 +937,41 @@ def pre_process_row(self, row, section): def to_zipped_csv(self, path, data, *args, **kwargs): def write_row(row, csv_writer, fields): - csv_writer.writerow( - [encode_if_str(row, field) for field in fields]) + csv_writer.writerow([encode_if_str(row, field) for field in fields]) csv_defs = {} - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") for section in self.sections: - csv_file = NamedTemporaryFile(suffix='.csv', mode='w') + csv_file = NamedTemporaryFile(suffix=".csv", mode="w") csv_writer = csv.writer(csv_file) - csv_defs[section['name']] = { - 'csv_file': csv_file, 'csv_writer': csv_writer} + csv_defs[section["name"]] = {"csv_file": csv_file, "csv_writer": csv_writer} # write headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "title") + csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'label') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "label") + csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") # write hxl row if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - hxl_row = [columns_with_hxl.get(col, '') - for col in fields] + fields = self.get_fields(dataview, section, "title") + hxl_row = [columns_with_hxl.get(col, "") for col in fields] if hxl_row: - writer = csv_defs[section['name']]['csv_writer'] + writer = csv_defs[section["name"]]["csv_writer"] writer.writerow(hxl_row) index = 1 @@ -794,10 +979,9 @@ def write_row(row, csv_writer, fields): survey_name = self.survey.name for i, d in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -807,86 +991,82 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] csv_def = csv_defs[section_name] - fields = self.get_fields(dataview, section, 'xpath') - csv_writer = csv_def['csv_writer'] + fields = self.get_fields(dataview, section, "xpath") + csv_writer = csv_def["csv_writer"] # section name might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - csv_writer, fields) + write_row(self.pre_process_row(row, section), csv_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - csv_writer, fields) + self.pre_process_row(child_row, section), csv_writer, fields + ) index += 1 track_task_progress(i, total_records) # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, csv_def) in iteritems(csv_defs): - csv_file = csv_def['csv_file'] + csv_file = csv_def["csv_file"] csv_file.seek(0) zip_file.write( - csv_file.name, '_'.join(section_name.split('/')) + '.csv') + csv_file.name, "_".join(section_name.split("/")) + ".csv" + ) # close files when we are done for (section_name, csv_def) in iteritems(csv_defs): - csv_def['csv_file'].close() + csv_def["csv_file"].close() @classmethod def get_valid_sheet_name(cls, desired_name, existing_names): # a sheet name has to be <= 31 characters and not a duplicate of an # existing sheet # truncate sheet_name to XLSDataFrameBuilder.SHEET_NAME_MAX_CHARS - new_sheet_name = \ - desired_name[:cls.XLS_SHEET_NAME_MAX_CHARS] + new_sheet_name = desired_name[: cls.XLS_SHEET_NAME_MAX_CHARS] # make sure its unique within the list i = 1 generated_name = new_sheet_name while generated_name in existing_names: digit_length = len(text(i)) - allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - \ - digit_length + allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - digit_length # make name the required len if len(generated_name) > allowed_name_len: generated_name = generated_name[:allowed_name_len] - generated_name = '{0}{1}'.format(generated_name, i) + generated_name = "{0}{1}".format(generated_name, i) i += 1 return generated_name def to_xls_export(self, path, data, *args, **kwargs): def write_row(data, work_sheet, fields, work_sheet_titles): # update parent_table with the generated sheet's title - data[PARENT_TABLE_NAME] = work_sheet_titles.get( - data.get(PARENT_TABLE_NAME)) + data[PARENT_TABLE_NAME] = work_sheet_titles.get(data.get(PARENT_TABLE_NAME)) work_sheet.append([data.get(f) for f in fields]) - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") wb = Workbook(write_only=True) work_sheets = {} # map of section_names to generated_names work_sheet_titles = {} for section in self.sections: - section_name = section['name'] + section_name = section["name"] work_sheet_title = ExportBuilder.get_valid_sheet_name( - '_'.join(section_name.split('/')), work_sheet_titles.values()) + "_".join(section_name.split("/")), work_sheet_titles.values() + ) work_sheet_titles[section_name] = work_sheet_title - work_sheets[section_name] = wb.create_sheet( - title=work_sheet_title) + work_sheets[section_name] = wb.create_sheet(title=work_sheet_title) # write the headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet ws = work_sheets[section_name] @@ -895,38 +1075,37 @@ def write_row(data, work_sheet, fields, work_sheet_titles): # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - labels = self.get_fields(dataview, section, 'label') + section_name = section["name"] + labels = self.get_fields(dataview, section, "label") # get the worksheet ws = work_sheets[section_name] ws.append(labels) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) # write hxl header - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet ws = work_sheets[section_name] - hxl_row = [columns_with_hxl.get(col, '') - for col in headers] + hxl_row = [columns_with_hxl.get(col, "") for col in headers] hxl_row and ws.append(hxl_row) index = 1 indices = {} survey_name = self.survey.name for i, d in enumerate(data, start=1): - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -936,8 +1115,8 @@ def write_row(data, work_sheet, fields, work_sheet_titles): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to xls - section_name = section['name'] - fields = self.get_fields(dataview, section, 'xpath') + section_name = section["name"] + fields = self.get_fields(dataview, section, "xpath") ws = work_sheets[section_name] # section might not exist within the output, e.g. data was @@ -946,60 +1125,79 @@ def write_row(data, work_sheet, fields, work_sheet_titles): if isinstance(row, dict): write_row( self.pre_process_row(row, section), - ws, fields, work_sheet_titles) + ws, + fields, + work_sheet_titles, + ) elif isinstance(row, list): for child_row in row: write_row( self.pre_process_row(child_row, section), - ws, fields, work_sheet_titles) + ws, + fields, + work_sheet_titles, + ) index += 1 track_task_progress(i, total_records) wb.save(filename=path) - def to_flat_csv_export(self, path, data, username, id_string, - filter_query, **kwargs): + def to_flat_csv_export( + self, path, data, username, id_string, filter_query, **kwargs + ): """ Generates a flattened CSV file for submitted data. """ # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder - start = kwargs.get('start') - end = kwargs.get('end') - dataview = kwargs.get('dataview') - xform = kwargs.get('xform') - options = kwargs.get('options') - total_records = kwargs.get('total_records') - win_excel_utf8 = options.get('win_excel_utf8') if options else False + + start = kwargs.get("start") + end = kwargs.get("end") + dataview = kwargs.get("dataview") + xform = kwargs.get("xform") + options = kwargs.get("options") + total_records = kwargs.get("total_records") + win_excel_utf8 = options.get("win_excel_utf8") if options else False index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS) - show_choice_labels = options.get('show_choice_labels', False) - language = options.get('language') + show_choice_labels = options.get("show_choice_labels", False) + language = options.get("language") csv_builder = CSVDataFrameBuilder( - username, id_string, filter_query, self.GROUP_DELIMITER, - self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, - start, end, self.TRUNCATE_GROUP_TITLE, xform, - self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, - self.INCLUDE_IMAGES, self.INCLUDE_HXL, - win_excel_utf8=win_excel_utf8, total_records=total_records, + username, + id_string, + filter_query, + self.GROUP_DELIMITER, + self.SPLIT_SELECT_MULTIPLES, + self.BINARY_SELECT_MULTIPLES, + start, + end, + self.TRUNCATE_GROUP_TITLE, + xform, + self.INCLUDE_LABELS, + self.INCLUDE_LABELS_ONLY, + self.INCLUDE_IMAGES, + self.INCLUDE_HXL, + win_excel_utf8=win_excel_utf8, + total_records=total_records, index_tags=index_tags, value_select_multiples=self.VALUE_SELECT_MULTIPLES, show_choice_labels=show_choice_labels, - include_reviews=self.INCLUDE_REVIEWS, language=language) + include_reviews=self.INCLUDE_REVIEWS, + language=language, + ) csv_builder.export_to(path, dataview=dataview) def get_default_language(self, languages): language = self.dd.default_language - if languages and \ - ((language and language not in languages) or not language): + if languages and ((language and language not in languages) or not language): languages.sort() language = languages[0] return language def _get_sav_value_labels(self, xpath_var_names=None): - """ GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form + """GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form `{varName: {value: valueLabel}}`: .. code-block: python @@ -1013,33 +1211,35 @@ def _get_sav_value_labels(self, xpath_var_names=None): sav_value_labels = {} for q in choice_questions: - if (xpath_var_names and - q.get_abbreviated_xpath() not in xpath_var_names): + if xpath_var_names and q.get_abbreviated_xpath() not in xpath_var_names: continue - var_name = xpath_var_names.get(q.get_abbreviated_xpath()) if \ - xpath_var_names else q['name'] - choices = q.to_json_dict().get('children') + var_name = ( + xpath_var_names.get(q.get_abbreviated_xpath()) + if xpath_var_names + else q["name"] + ) + choices = q.to_json_dict().get("children") if choices is None: - choices = self.survey.get('choices') - if choices is not None and q.get('itemset'): - choices = choices.get(q.get('itemset')) + choices = self.survey.get("choices") + if choices is not None and q.get("itemset"): + choices = choices.get(q.get("itemset")) _value_labels = {} if choices: - is_numeric = is_all_numeric([c['name'] for c in choices]) + is_numeric = is_all_numeric([c["name"] for c in choices]) for choice in choices: - name = choice['name'].strip() + name = choice["name"].strip() # should skip select multiple and zero padded numbers e.g # 009 or 09, they should be treated as strings - if q.type != 'select all that apply' and is_numeric: + if q.type != "select all that apply" and is_numeric: try: - name = float(name) \ - if (float(name) > int(name)) else int(name) + name = ( + float(name) if (float(name) > int(name)) else int(name) + ) except ValueError: pass - label = self.get_choice_label_from_dict( - choice.get("label", "")) + label = self.get_choice_label_from_dict(choice.get("label", "")) _value_labels[name] = label.strip() - sav_value_labels[var_name or q['name']] = _value_labels + sav_value_labels[var_name or q["name"]] = _value_labels return sav_value_labels @@ -1050,10 +1250,15 @@ def _get_var_name(self, title, var_names): @param var_names - list of existing var_names @return valid varName and list of var_names with new var name appended """ - var_name = title.replace('/', '.').replace('-', '_'). \ - replace(':', '_').replace('{', '').replace('}', '') + var_name = ( + title.replace("/", ".") + .replace("-", "_") + .replace(":", "_") + .replace("{", "") + .replace("}", "") + ) var_name = self._check_sav_column(var_name, var_names) - var_name = '@' + var_name if var_name.startswith('_') else var_name + var_name = "@" + var_name if var_name.startswith("_") else var_name var_names.append(var_name) return var_name, var_names @@ -1071,11 +1276,12 @@ def _get_sav_options(self, elements): 'ioUtf8': True } """ + def _is_numeric(xpath, element_type, data_dictionary): var_name = xpath_var_names.get(xpath) or xpath - if element_type in ['decimal', 'int', 'date']: + if element_type in ["decimal", "int", "date"]: return True - elif element_type == 'string': + elif element_type == "string": # check if it is a choice part of multiple choice # type is likely empty string, split multi select is binary element = data_dictionary.get_element(xpath) @@ -1087,11 +1293,11 @@ def _is_numeric(xpath, element_type, data_dictionary): if len(choices) == 0: return False return is_all_numeric(choices) - if element and element.type == '' and value_select_multiples: + if element and element.type == "" and value_select_multiples: return is_all_numeric([element.name]) - parent_xpath = '/'.join(xpath.split('/')[:-1]) + parent_xpath = "/".join(xpath.split("/")[:-1]) parent = data_dictionary.get_element(parent_xpath) - return (parent and parent.type == MULTIPLE_SELECT_TYPE) + return parent and parent.type == MULTIPLE_SELECT_TYPE else: return False @@ -1102,18 +1308,20 @@ def _is_numeric(xpath, element_type, data_dictionary): var_names = [] fields_and_labels = [] - elements += [{'title': f, "label": f, "xpath": f, 'type': f} - for f in self.extra_columns] + elements += [ + {"title": f, "label": f, "xpath": f, "type": f} for f in self.extra_columns + ] for element in elements: - title = element['title'] + title = element["title"] _var_name, _var_names = self._get_var_name(title, var_names) var_names = _var_names - fields_and_labels.append((element['title'], element['label'], - element['xpath'], _var_name)) + fields_and_labels.append( + (element["title"], element["label"], element["xpath"], _var_name) + ) - xpath_var_names = dict([(xpath, var_name) - for field, label, xpath, var_name - in fields_and_labels]) + xpath_var_names = dict( + [(xpath, var_name) for field, label, xpath, var_name in fields_and_labels] + ) all_value_labels = self._get_sav_value_labels(xpath_var_names) @@ -1138,36 +1346,49 @@ def _get_element_type(element_xpath): return "" var_types = dict( - [(_var_types[element['xpath']], - SAV_NUMERIC_TYPE if _is_numeric(element['xpath'], - element['type'], - self.dd) else - SAV_255_BYTES_TYPE) - for element in elements] + - [(_var_types[item], - SAV_NUMERIC_TYPE if item in [ - '_id', '_index', '_parent_index', SUBMISSION_TIME] - else SAV_255_BYTES_TYPE) - for item in self.extra_columns] + - [(x[1], - SAV_NUMERIC_TYPE if _is_numeric( - x[0], - _get_element_type(x[0]), - self.dd) else SAV_255_BYTES_TYPE) - for x in duplicate_names] + [ + ( + _var_types[element["xpath"]], + SAV_NUMERIC_TYPE + if _is_numeric(element["xpath"], element["type"], self.dd) + else SAV_255_BYTES_TYPE, + ) + for element in elements + ] + + [ + ( + _var_types[item], + SAV_NUMERIC_TYPE + if item in ["_id", "_index", "_parent_index", SUBMISSION_TIME] + else SAV_255_BYTES_TYPE, + ) + for item in self.extra_columns + ] + + [ + ( + x[1], + SAV_NUMERIC_TYPE + if _is_numeric(x[0], _get_element_type(x[0]), self.dd) + else SAV_255_BYTES_TYPE, + ) + for x in duplicate_names + ] ) - dates = [_var_types[element['xpath']] for element in elements - if element.get('type') == 'date'] - formats = {d: 'EDATE40' for d in dates} - formats['@' + SUBMISSION_TIME] = 'DATETIME40' + dates = [ + _var_types[element["xpath"]] + for element in elements + if element.get("type") == "date" + ] + formats = {d: "EDATE40" for d in dates} + formats["@" + SUBMISSION_TIME] = "DATETIME40" return { - 'formats': formats, - 'varLabels': var_labels, - 'varNames': var_names, - 'varTypes': var_types, - 'valueLabels': value_labels, - 'ioUtf8': True + "formats": formats, + "varLabels": var_labels, + "varNames": var_names, + "varTypes": var_types, + "valueLabels": value_labels, + "ioUtf8": True, } def _check_sav_column(self, column, columns): @@ -1185,43 +1406,43 @@ def _check_sav_column(self, column, columns): if column.lower() in (t.lower() for t in columns): if len(column) > 59: column = column[:-5] - column = column + '@' + text(uuid.uuid4()).split('-')[1] + column = column + "@" + text(uuid.uuid4()).split("-")[1] return column def to_zipped_sav(self, path, data, *args, **kwargs): - total_records = kwargs.get('total_records') + total_records = kwargs.get("total_records") def write_row(row, csv_writer, fields): # replace character for osm fields - fields = [field.replace(':', '_') for field in fields] + fields = [field.replace(":", "_") for field in fields] sav_writer.writerow( - [encode_if_str(row, field, sav_writer=sav_writer) - for field in fields]) + [encode_if_str(row, field, sav_writer=sav_writer) for field in fields] + ) sav_defs = {} # write headers for section in self.sections: - sav_options = self._get_sav_options(section['elements']) - sav_file = NamedTemporaryFile(suffix='.sav') - sav_writer = SavWriter(sav_file.name, ioLocale=str('en_US.UTF-8'), - **sav_options) - sav_defs[section['name']] = { - 'sav_file': sav_file, 'sav_writer': sav_writer} + sav_options = self._get_sav_options(section["elements"]) + sav_file = NamedTemporaryFile(suffix=".sav") + sav_writer = SavWriter( + sav_file.name, ioLocale=str("en_US.UTF-8"), **sav_options + ) + sav_defs[section["name"]] = {"sav_file": sav_file, "sav_writer": sav_writer} - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) index = 1 indices = {} survey_name = self.survey.name for i, d in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -1231,52 +1452,53 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] sav_def = sav_defs[section_name] - fields = [ - element['xpath'] for element in - section['elements']] - sav_writer = sav_def['sav_writer'] + fields = [element["xpath"] for element in section["elements"]] + sav_writer = sav_def["sav_writer"] row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - sav_writer, fields) + write_row(self.pre_process_row(row, section), sav_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - sav_writer, fields) + self.pre_process_row(child_row, section), sav_writer, fields + ) index += 1 track_task_progress(i, total_records) for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_writer'].closeSavFile( - sav_def['sav_writer'].fh, mode='wb') + sav_def["sav_writer"].closeSavFile(sav_def["sav_writer"].fh, mode="wb") # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, sav_def) in iteritems(sav_defs): - sav_file = sav_def['sav_file'] + sav_file = sav_def["sav_file"] sav_file.seek(0) zip_file.write( - sav_file.name, '_'.join(section_name.split('/')) + '.sav') + sav_file.name, "_".join(section_name.split("/")) + ".sav" + ) # close files when we are done for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_file'].close() + sav_def["sav_file"].close() def get_fields(self, dataview, section, key): """ Return list of element value with the key in section['elements']. """ if dataview: - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements'] - if element['title'] in dataview.columns] + \ - self.extra_columns - - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements']] + self.extra_columns + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + if element["title"] in dataview.columns + ] + self.extra_columns + + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + ] + self.extra_columns diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a05b32e60c..f4ba38bd7e 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -22,8 +22,8 @@ from django.shortcuts import render from django.utils import timezone from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import iteritems +from six.moves.urllib.parse import urlparse +from six import iteritems from json2xlsclient.client import Client from rest_framework import exceptions from savReaderWriter import SPSSIOError @@ -32,27 +32,25 @@ from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm from onadata.apps.logger.models.data_view import DataView from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.viewer.models.export import (Export, - get_export_options_query_kwargs) +from onadata.apps.viewer.models.export import Export, get_export_options_query_kwargs from onadata.apps.viewer.models.parsed_instance import query_data from onadata.libs.exceptions import J2XException, NoRecordsFoundError -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG) -from onadata.libs.utils.common_tools import (str_to_bool, - cmp_to_key, - report_exception, - retry) +from onadata.libs.utils.common_tags import DATAVIEW_EXPORT, GROUPNAME_REMOVED_FLAG +from onadata.libs.utils.common_tools import ( + str_to_bool, + cmp_to_key, + report_exception, + retry, +) from onadata.libs.utils.export_builder import ExportBuilder -from onadata.libs.utils.model_tools import (get_columns_with_hxl, - queryset_iterator) +from onadata.libs.utils.model_tools import get_columns_with_hxl, queryset_iterator from onadata.libs.utils.osm import get_combined_osm -from onadata.libs.utils.viewer_tools import (create_attachments_zipfile, - image_urls) +from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls -DEFAULT_GROUP_DELIMITER = '/' -DEFAULT_INDEX_TAGS = ('[', ']') -SUPPORTED_INDEX_TAGS = ('[', ']', '(', ')', '{', '}', '.', '_') -EXPORT_QUERY_KEY = 'query' +DEFAULT_GROUP_DELIMITER = "/" +DEFAULT_INDEX_TAGS = ("[", "]") +SUPPORTED_INDEX_TAGS = ("[", "]", "(", ")", "{", "}", ".", "_") +EXPORT_QUERY_KEY = "query" MAX_RETRIES = 3 @@ -69,8 +67,10 @@ def get_export_options(options): list of provided options to be saved with each Export object. """ export_options = { - key: value for (key, value) in iteritems(options) - if key in Export.EXPORT_OPTION_FIELDS} + key: value + for (key, value) in iteritems(options) + if key in Export.EXPORT_OPTION_FIELDS + } return export_options @@ -84,7 +84,7 @@ def get_or_create_export(export_id, xform, export_type, options): try: return Export.objects.get(pk=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): + if getattr(settings, "SLAVE_DATABASES", []): from multidb.pinning import use_master with use_master: @@ -127,30 +127,32 @@ def generate_export(export_type, xform, export_id=None, options=None): start = options.get("start") export_type_func_map = { - Export.XLS_EXPORT: 'to_xls_export', - Export.CSV_EXPORT: 'to_flat_csv_export', - Export.CSV_ZIP_EXPORT: 'to_zipped_csv', - Export.SAV_ZIP_EXPORT: 'to_zipped_sav', - Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', + Export.XLS_EXPORT: "to_xls_export", + Export.CSV_EXPORT: "to_flat_csv_export", + Export.CSV_ZIP_EXPORT: "to_zipped_csv", + Export.SAV_ZIP_EXPORT: "to_zipped_sav", + Export.GOOGLE_SHEETS_EXPORT: "to_google_sheets", } if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) - records = dataview.query_data(dataview, all_data=True, - filter_query=filter_query) - total_records = dataview.query_data(dataview, - count=True)[0].get('count') + records = dataview.query_data( + dataview, all_data=True, filter_query=filter_query + ) + total_records = dataview.query_data(dataview, count=True)[0].get("count") else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: - total_records = query_data(xform, query=filter_query, start=start, - end=end, count=True)[0].get('count') + total_records = query_data( + xform, query=filter_query, start=start, end=end, count=True + )[0].get("count") else: total_records = xform.num_of_submissions @@ -158,59 +160,63 @@ def generate_export(export_type, xform, export_id=None, options=None): records = records.iterator() export_builder = ExportBuilder() - export_builder.TRUNCATE_GROUP_TITLE = True \ - if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + export_builder.TRUNCATE_GROUP_TITLE = ( + True if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + ) export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) - export_builder.SPLIT_SELECT_MULTIPLES = options.get( - "split_select_multiples", True - ) + export_builder.SPLIT_SELECT_MULTIPLES = options.get("split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) - export_builder.INCLUDE_LABELS = options.get('include_labels', False) - include_reviews = options.get('include_reviews', False) - export_builder.INCLUDE_LABELS_ONLY = options.get( - 'include_labels_only', False - ) - export_builder.INCLUDE_HXL = options.get('include_hxl', False) + export_builder.INCLUDE_LABELS = options.get("include_labels", False) + include_reviews = options.get("include_reviews", False) + export_builder.INCLUDE_LABELS_ONLY = options.get("include_labels_only", False) + export_builder.INCLUDE_HXL = options.get("include_hxl", False) - export_builder.INCLUDE_IMAGES \ - = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) + export_builder.INCLUDE_IMAGES = options.get( + "include_images", settings.EXPORT_WITH_IMAGE_DEFAULT + ) - export_builder.VALUE_SELECT_MULTIPLES = options.get( - 'value_select_multiples', False) + export_builder.VALUE_SELECT_MULTIPLES = options.get("value_select_multiples", False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) - export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', - False) + export_builder.SHOW_CHOICE_LABELS = options.get("show_choice_labels", False) - export_builder.language = options.get('language') + export_builder.language = options.get("language") # 'win_excel_utf8' is only relevant for CSV exports - if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: - del options['win_excel_utf8'] + if "win_excel_utf8" in options and export_type != Export.CSV_EXPORT: + del options["win_excel_utf8"] export_builder.INCLUDE_REVIEWS = include_reviews - export_builder.set_survey(xform.survey, xform, - include_reviews=include_reviews) + export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( - xform.survey_elements) + xform.survey_elements + ) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( - temp_file.name, records, username, id_string, filter_query, - start=start, end=end, dataview=dataview, xform=xform, - options=options, columns_with_hxl=columns_with_hxl, - total_records=total_records + temp_file.name, + records, + username, + id_string, + filter_query, + start=start, + end=end, + dataview=dataview, + xform=xform, + options=options, + columns_with_hxl=columns_with_hxl, + total_records=total_records, ) except NoRecordsFoundError: pass @@ -223,8 +229,7 @@ def generate_export(export_type, xform, export_id=None, options=None): return export # generate filename - basename = "%s_%s" % ( - id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename @@ -238,17 +243,11 @@ def generate_export(export_type, xform, export_id=None, options=None): while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) - export_filename = default_storage.save(file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) @@ -275,20 +274,25 @@ def create_export_object(xform, export_type, options): Return an export object that has not been saved to the database. """ export_options = get_export_options(options) - return Export(xform=xform, export_type=export_type, options=export_options, - created_on=timezone.now()) + return Export( + xform=xform, + export_type=export_type, + options=export_options, + created_on=timezone.now(), + ) -def check_pending_export(xform, export_type, options, - minutes=getattr(settings, 'PENDING_EXPORT_TIME', 5)): +def check_pending_export( + xform, export_type, options, minutes=getattr(settings, "PENDING_EXPORT_TIME", 5) +): """ - Check for pending export done within a specific period of time and - returns the export - :param xform: - :param export_type: - :param options: - :param minutes - :return: + Check for pending export done within a specific period of time and + returns the export + :param xform: + :param export_type: + :param options: + :param minutes + :return: """ created_time = timezone.now() - timedelta(minutes=minutes) export_options_kwargs = get_export_options_query_kwargs(options) @@ -303,10 +307,7 @@ def check_pending_export(xform, export_type, options, return export -def should_create_new_export(xform, - export_type, - options, - request=None): +def should_create_new_export(xform, export_type, options, request=None): """ Function that determines whether to create a new export. param: xform @@ -319,27 +320,27 @@ def should_create_new_export(xform, index_tag: ('[', ']') or ('_', '_') params: request: Get params are used to determine if new export is required """ - split_select_multiples = options.get('split_select_multiples', True) + split_select_multiples = options.get("split_select_multiples", True) - if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False): + if getattr(settings, "SHOULD_ALWAYS_CREATE_NEW_EXPORT", False): return True - if (request and (frozenset(list(request.GET)) & - frozenset(['start', 'end', 'data_id']))) or\ - not split_select_multiples: + if ( + request + and (frozenset(list(request.GET)) & frozenset(["start", "end", "data_id"])) + ) or not split_select_multiples: return True export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) if options.get(EXPORT_QUERY_KEY) is None: export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY) - if export_query.count() == 0 or\ - Export.exports_outdated(xform, export_type, options=options): + if export_query.count() == 0 or Export.exports_outdated( + xform, export_type, options=options + ): return True return False @@ -361,12 +362,10 @@ def newest_export_for(xform, export_type, options): export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) - return export_query.latest('created_on') + return export_query.latest("created_on") def increment_index_in_filename(filename): @@ -390,9 +389,9 @@ def increment_index_in_filename(filename): # pylint: disable=R0913 -def generate_attachments_zip_export(export_type, username, id_string, - export_id=None, options=None, - xform=None): +def generate_attachments_zip_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates zip export of attachments. @@ -413,45 +412,44 @@ def generate_attachments_zip_export(export_type, username, id_string, dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ - rec.get('_id') + rec.get("_id") for rec in dataview.query_data( - dataview, all_data=True, filter_query=filter_query)], - instance__deleted_at__isnull=True) + dataview, all_data=True, filter_query=filter_query + ) + ], + instance__deleted_at__isnull=True, + ) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) - attachments = Attachment.objects.filter( - instance__deleted_at__isnull=True) + attachments = Attachment.objects.filter(instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( instance__xform_id__in=[ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', flat=True)]).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] + ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) else: - attachments = attachments.filter( - instance__xform_id=xform.pk).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) - - filename = "%s_%s.%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), - export_type.lower()) - file_path = os.path.join( - username, - 'exports', + attachments = attachments.filter(instance__xform_id=xform.pk).filter( + instance_id__in=[i_id["_id"] for i_id in instance_ids] + ) + + filename = "%s_%s.%s" % ( id_string, - export_type, - filename) + datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), + export_type.lower(), + ) + file_path = os.path.join(username, "exports", id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: - temp_file = builtins.open(zip_file.name, 'rb') - filename = default_storage.save( - file_path, - File(temp_file, file_path)) + temp_file = builtins.open(zip_file.name, "rb") + filename = default_storage.save(file_path, File(temp_file, file_path)) finally: temp_file.close() finally: @@ -467,7 +465,7 @@ def generate_attachments_zip_export(export_type, username, id_string, def write_temp_file_to_path(suffix, content, file_path): - """ Write a temp file and return the name of the file. + """Write a temp file and return the name of the file. :param suffix: The file suffix :param content: The content to write :param file_path: The path to write the temp file to @@ -476,16 +474,14 @@ def write_temp_file_to_path(suffix, content, file_path): temp_file = NamedTemporaryFile(suffix=suffix) temp_file.write(content) temp_file.seek(0) - export_filename = default_storage.save( - file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() return export_filename def get_or_create_export_object(export_id, options, xform, export_type): - """ Get or create export object. + """Get or create export object. :param export_id: Export ID :param options: Options to convert to export options @@ -497,16 +493,17 @@ def get_or_create_export_object(export_id, options, xform, export_type): export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) return export # pylint: disable=R0913 -def generate_kml_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_kml_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates kml export for geographical data @@ -524,25 +521,18 @@ def generate_kml_export(export_type, username, id_string, export_id=None, xform = XForm.objects.get(user__username=username, id_string=id_string) response = render( - None, 'survey.kml', - {'data': kml_export_data(id_string, user, xform=xform)} + None, "survey.kml", {"data": kml_export_data(id_string, user, xform=xform)} ) - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + export_type.lower() - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path( - export_type.lower(), response.content, file_path) + export_type.lower(), response.content, file_path + ) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) export.filedir, export.filename = os.path.split(export_filename) export.internal_status = Export.SUCCESSFUL @@ -555,6 +545,7 @@ def kml_export_data(id_string, user, xform=None): """ KML export data from form submissions. """ + def cached_get_labels(xpath): """ Get and Cache labels for the XForm. @@ -567,16 +558,21 @@ def cached_get_labels(xpath): xform = xform or XForm.objects.get(id_string=id_string, user=user) - data_kwargs = {'geom__isnull': False} + data_kwargs = {"geom__isnull": False} if xform.is_merged_dataset: - data_kwargs.update({ - 'xform_id__in': - [i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] - }) + data_kwargs.update( + { + "xform_id__in": [ + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] + } + ) else: - data_kwargs.update({'xform_id': xform.pk}) - instances = Instance.objects.filter(**data_kwargs).order_by('id') + data_kwargs.update({"xform_id": xform.pk}) + instances = Instance.objects.filter(**data_kwargs).order_by("id") data_for_template = [] labels = {} for instance in queryset_iterator(instances): @@ -585,22 +581,26 @@ def cached_get_labels(xpath): xpaths = list(data_for_display) xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp())) table_rows = [ - '%s%s' % - (cached_get_labels(xpath), data_for_display[xpath]) for xpath in - xpaths if not xpath.startswith(u"_")] + "%s%s" + % (cached_get_labels(xpath), data_for_display[xpath]) + for xpath in xpaths + if not xpath.startswith("_") + ] img_urls = image_urls(instance) if instance.point: - data_for_template.append({ - 'name': instance.xform.id_string, - 'id': instance.id, - 'lat': instance.point.y, - 'lng': instance.point.x, - 'image_urls': img_urls, - 'table': '%s' - '
    ' % (img_urls[0] if img_urls else "", - ''.join(table_rows))}) + data_for_template.append( + { + "name": instance.xform.id_string, + "id": instance.id, + "lat": instance.point.y, + "lng": instance.point.x, + "image_urls": img_urls, + "table": '%s' + "
    " % (img_urls[0] if img_urls else "", "".join(table_rows)), + } + ) return data_for_template @@ -608,20 +608,24 @@ def cached_get_labels(xpath): def get_osm_data_kwargs(xform): """Return kwargs for OsmData queryset for given xform""" - kwargs = {'instance__deleted_at__isnull': True} + kwargs = {"instance__deleted_at__isnull": True} if xform.is_merged_dataset: - kwargs['instance__xform_id__in'] = [ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] + kwargs["instance__xform_id__in"] = [ + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] else: - kwargs['instance__xform_id'] = xform.pk + kwargs["instance__xform_id"] = xform.pk return kwargs -def generate_osm_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_osm_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates osm export for OpenStreetMap data @@ -642,20 +646,13 @@ def generate_osm_export(export_type, username, id_string, export_id=None, osm_list = OsmData.objects.filter(**kwargs) content = get_combined_osm(osm_list) - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path(extension, content, file_path) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) dir_name, basename = os.path.split(export_filename) export.filedir = dir_name @@ -667,8 +664,7 @@ def generate_osm_export(export_type, username, id_string, export_id=None, def _get_records(instances): - return [clean_keys_of_slashes(instance) - for instance in instances] + return [clean_keys_of_slashes(instance) for instance in instances] def clean_keys_of_slashes(record): @@ -679,10 +675,9 @@ def clean_keys_of_slashes(record): """ for key in list(record): value = record[key] - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply same if value: if isinstance(value, list) and isinstance(value[0], dict): @@ -698,6 +693,7 @@ def _get_server_from_metadata(xform, meta, token): report_templates = MetaData.external_export(xform) except MetaData.DoesNotExist: from multidb.pinning import use_master + with use_master: report_templates = MetaData.external_export(xform) @@ -705,7 +701,7 @@ def _get_server_from_metadata(xform, meta, token): try: int(meta) except ValueError: - raise Exception(u"Invalid metadata pk {0}".format(meta)) + raise Exception("Invalid metadata pk {0}".format(meta)) # Get the external server from the metadata result = report_templates.get(pk=meta) @@ -718,7 +714,8 @@ def _get_server_from_metadata(xform, meta, token): # Take the latest value in the metadata if not report_templates: raise Exception( - u"Could not find the template token: Please upload template.") + "Could not find the template token: Please upload template." + ) server = report_templates[0].external_export_url name = report_templates[0].external_export_name @@ -726,8 +723,9 @@ def _get_server_from_metadata(xform, meta, token): return server, name -def generate_external_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_external_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates external export using ONA data through an external service. @@ -748,7 +746,8 @@ def generate_external_export(export_type, username, id_string, export_id=None, if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) @@ -758,14 +757,13 @@ def generate_external_export(export_type, username, id_string, export_id=None, token = parsed_url.path[5:] - ser = parsed_url.scheme + '://' + parsed_url.netloc + ser = parsed_url.scheme + "://" + parsed_url.netloc # Get single submission data if data_id: - inst = Instance.objects.filter(xform__user=user, - xform__id_string=id_string, - deleted_at=None, - pk=data_id) + inst = Instance.objects.filter( + xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id + ) instances = [inst[0].json if inst else {}] else: @@ -780,20 +778,18 @@ def generate_external_export(export_type, username, id_string, export_id=None, client = Client(ser) response = client.xls.create(token, json.dumps(records)) - if hasattr(client.xls.conn, 'last_response'): + if hasattr(client.xls.conn, "last_response"): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( - u"J2X client could not generate report. Server -> {0}," - u" Error-> {1}".format(server, e) + "J2X client could not generate report. Server -> {0}," + " Error-> {1}".format(server, e) ) else: if not server: - raise J2XException(u"External server not set") + raise J2XException("External server not set") elif not records: - raise J2XException( - u"No record to export. Form -> {0}".format(id_string) - ) + raise J2XException("No record to export. Form -> {0}".format(id_string)) # get or create export object if export_id: @@ -804,14 +800,14 @@ def generate_external_export(export_type, username, id_string, export_id=None, export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL - export.filename = name + '-' + response[5:] if name else response[5:] + export.filename = name + "-" + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED @@ -832,10 +828,10 @@ def upload_template_for_external_export(server, file_obj): response = client.template.create(template_file=file_obj) status_code = None - if hasattr(client.template.conn, 'last_response'): + if hasattr(client.template.conn, "last_response"): status_code = client.template.conn.last_response.status_code - return str(status_code) + '|' + response + return str(status_code) + "|" + response def parse_request_export_options(params): # pylint: disable=too-many-branches @@ -845,71 +841,73 @@ def parse_request_export_options(params): # pylint: disable=too-many-branches removed, the group delimiter, and a boolean for whether select multiples should be split. """ - boolean_list = ['true', 'false'] + boolean_list = ["true", "false"] options = {} - remove_group_name = params.get('remove_group_name') and \ - params.get('remove_group_name').lower() - binary_select_multiples = params.get('binary_select_multiples') and \ - params.get('binary_select_multiples').lower() - do_not_split_select_multiples = params.get( - 'do_not_split_select_multiples') - include_labels = params.get('include_labels', False) - include_reviews = params.get('include_reviews', False) - include_labels_only = params.get('include_labels_only', False) - include_hxl = params.get('include_hxl', True) - value_select_multiples = params.get('value_select_multiples') and \ - params.get('value_select_multiples').lower() - show_choice_labels = params.get('show_choice_labels') and \ - params.get('show_choice_labels').lower() + remove_group_name = ( + params.get("remove_group_name") and params.get("remove_group_name").lower() + ) + binary_select_multiples = ( + params.get("binary_select_multiples") + and params.get("binary_select_multiples").lower() + ) + do_not_split_select_multiples = params.get("do_not_split_select_multiples") + include_labels = params.get("include_labels", False) + include_reviews = params.get("include_reviews", False) + include_labels_only = params.get("include_labels_only", False) + include_hxl = params.get("include_hxl", True) + value_select_multiples = ( + params.get("value_select_multiples") + and params.get("value_select_multiples").lower() + ) + show_choice_labels = ( + params.get("show_choice_labels") and params.get("show_choice_labels").lower() + ) if include_labels is not None: - options['include_labels'] = str_to_bool(include_labels) + options["include_labels"] = str_to_bool(include_labels) if include_reviews is not None: - options['include_reviews'] = str_to_bool(include_reviews) + options["include_reviews"] = str_to_bool(include_reviews) if include_labels_only is not None: - options['include_labels_only'] = str_to_bool(include_labels_only) + options["include_labels_only"] = str_to_bool(include_labels_only) if include_hxl is not None: - options['include_hxl'] = str_to_bool(include_hxl) + options["include_hxl"] = str_to_bool(include_hxl) if remove_group_name in boolean_list: options["remove_group_name"] = str_to_bool(remove_group_name) else: options["remove_group_name"] = False - if params.get("group_delimiter") in ['.', DEFAULT_GROUP_DELIMITER]: - options['group_delimiter'] = params.get("group_delimiter") + if params.get("group_delimiter") in [".", DEFAULT_GROUP_DELIMITER]: + options["group_delimiter"] = params.get("group_delimiter") else: - options['group_delimiter'] = DEFAULT_GROUP_DELIMITER + options["group_delimiter"] = DEFAULT_GROUP_DELIMITER - options['split_select_multiples'] = \ - not str_to_bool(do_not_split_select_multiples) + options["split_select_multiples"] = not str_to_bool(do_not_split_select_multiples) if binary_select_multiples and binary_select_multiples in boolean_list: - options['binary_select_multiples'] = str_to_bool( - binary_select_multiples) + options["binary_select_multiples"] = str_to_bool(binary_select_multiples) - if 'include_images' in params: - options["include_images"] = str_to_bool( - params.get("include_images")) + if "include_images" in params: + options["include_images"] = str_to_bool(params.get("include_images")) else: options["include_images"] = settings.EXPORT_WITH_IMAGE_DEFAULT - options['win_excel_utf8'] = str_to_bool(params.get('win_excel_utf8')) + options["win_excel_utf8"] = str_to_bool(params.get("win_excel_utf8")) if value_select_multiples and value_select_multiples in boolean_list: - options['value_select_multiples'] = str_to_bool(value_select_multiples) + options["value_select_multiples"] = str_to_bool(value_select_multiples) if show_choice_labels and show_choice_labels in boolean_list: - options['show_choice_labels'] = str_to_bool(show_choice_labels) + options["show_choice_labels"] = str_to_bool(show_choice_labels) index_tags = get_repeat_index_tags(params.get("repeat_index_tags")) if index_tags: - options['repeat_index_tags'] = index_tags + options["repeat_index_tags"] = index_tags - if 'language' in params: - options['language'] = params.get('language') + if "language" in params: + options["language"] = params.get("language") return options @@ -921,7 +919,7 @@ def get_repeat_index_tags(index_tags): Retuns a tuple of two strings with SUPPORTED_INDEX_TAGS, """ if isinstance(index_tags, six.string_types): - index_tags = tuple(index_tags.split(',')) + index_tags = tuple(index_tags.split(",")) length = len(index_tags) if length == 1: index_tags = (index_tags[0], index_tags[0]) @@ -932,8 +930,11 @@ def get_repeat_index_tags(index_tags): for tag in index_tags: if tag not in SUPPORTED_INDEX_TAGS: - raise exceptions.ParseError(_( - "The tag %s is not supported, supported tags are %s" % - (tag, SUPPORTED_INDEX_TAGS))) + raise exceptions.ParseError( + _( + "The tag %s is not supported, supported tags are %s" + % (tag, SUPPORTED_INDEX_TAGS) + ) + ) return index_tags diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index afe3b0a18e..bae078432d 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,6 +1,6 @@ import hashlib -from future.moves.urllib.parse import urlencode -from future.moves.urllib.request import urlopen +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen DEFAULT_GRAVATAR = "https://ona.io/static/images/default_avatar.png" GRAVATAR_ENDPOINT = "https://secure.gravatar.com/avatar/" @@ -8,12 +8,16 @@ def email_md5(user): - return hashlib.md5(user.email.lower().encode('utf-8')).hexdigest() + return hashlib.md5(user.email.lower().encode("utf-8")).hexdigest() def get_gravatar_img_link(user): - return GRAVATAR_ENDPOINT + email_md5(user) + "?" + urlencode({ - 'd': DEFAULT_GRAVATAR, 's': str(GRAVATAR_SIZE)}) + return ( + GRAVATAR_ENDPOINT + + email_md5(user) + + "?" + + urlencode({"d": DEFAULT_GRAVATAR, "s": str(GRAVATAR_SIZE)}) + ) def gravatar_exists(user): diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 23393476c1..f4a1c17839 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -6,11 +6,10 @@ import logging -from django.contrib.gis.geos import (GeometryCollection, LineString, Point, - Polygon) +from django.contrib.gis.geos import GeometryCollection, LineString, Point, Polygon from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from future.utils import iteritems +from six import iteritems from lxml import etree from onadata.apps.logger.models.attachment import Attachment @@ -26,8 +25,8 @@ def _get_xml_obj(xml): try: return etree.fromstring(xml) # pylint: disable=E1101 except etree.XMLSyntaxError as e: # pylint: disable=E1101 - if 'Attribute action redefined' in e.msg: - xml = xml.replace(b'action="modify" ', b'') + if "Attribute action redefined" in e.msg: + xml = xml.replace(b'action="modify" ', b"") return _get_xml_obj(xml) @@ -37,7 +36,7 @@ def _get_node(ref, root): nodes = root.xpath('//node[@id="{}"]'.format(ref)) if nodes: node = nodes[0] - point = Point(float(node.get('lon')), float(node.get('lat'))) + point = Point(float(node.get("lon")), float(node.get("lat"))) return point @@ -46,9 +45,10 @@ def get_combined_osm(osm_list): """ Combine osm xml form list of OsmData objects """ - xml = '' - if (osm_list and isinstance(osm_list, list)) \ - or isinstance(osm_list, models.QuerySet): + xml = "" + if (osm_list and isinstance(osm_list, list)) or isinstance( + osm_list, models.QuerySet + ): osm = None for osm_data in osm_list: osm_xml = osm_data.xml @@ -62,65 +62,59 @@ def get_combined_osm(osm_list): osm.append(child) if osm is not None: # pylint: disable=E1101 - return etree.tostring(osm, encoding='utf-8', xml_declaration=True) + return etree.tostring(osm, encoding="utf-8", xml_declaration=True) elif isinstance(osm_list, dict): - if 'detail' in osm_list: - xml = '%s' % osm_list['detail'] - return xml.encode('utf-8') + if "detail" in osm_list: + xml = "%s" % osm_list["detail"] + return xml.encode("utf-8") def parse_osm_ways(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for way in root.findall('way'): + for way in root.findall("way"): geom = None points = [] - for node in way.findall('nd'): - points.append(_get_node(node.get('ref'), root)) + for node in way.findall("nd"): + points.append(_get_node(node.get("ref"), root)) try: geom = Polygon(points) except GEOSException: geom = LineString(points) tags = parse_osm_tags(way, include_osm_id) - items.append({ - 'osm_id': way.get('id'), - 'geom': geom, - 'tags': tags, - 'osm_type': 'way' - }) + items.append( + {"osm_id": way.get("id"), "geom": geom, "tags": tags, "osm_type": "way"} + ) return items def parse_osm_nodes(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for node in root.findall('node'): - point = Point(float(node.get('lon')), float(node.get('lat'))) + for node in root.findall("node"): + point = Point(float(node.get("lon")), float(node.get("lat"))) tags = parse_osm_tags(node, include_osm_id) - items.append({ - 'osm_id': node.get('id'), - 'geom': point, - 'tags': tags, - 'osm_type': 'node' - }) + items.append( + {"osm_id": node.get("id"), "geom": point, "tags": tags, "osm_type": "node"} + ) return items def parse_osm_tags(node, include_osm_id=False): """Retrieves all the tags from a osm xml node""" - tags = {} if not include_osm_id else {node.tag + ':id': node.get('id')} - for tag in node.findall('tag'): - key, val = tag.attrib['k'], tag.attrib['v'] - if val == '' or val.upper() == 'FIXME': + tags = {} if not include_osm_id else {node.tag + ":id": node.get("id")} + for tag in node.findall("tag"): + key, val = tag.attrib["k"], tag.attrib["v"] + if val == "" or val.upper() == "FIXME": continue tags.update({key: val}) @@ -153,24 +147,24 @@ def save_osm_data(instance_id): Includes the OSM data in the specified submission json data. """ instance = Instance.objects.filter(pk=instance_id).first() - osm_attachments = instance.attachments.filter(extension=Attachment.OSM) \ - if instance else None + osm_attachments = ( + instance.attachments.filter(extension=Attachment.OSM) if instance else None + ) if instance and osm_attachments: fields = [ f.get_abbreviated_xpath() - for f in instance.xform.get_survey_elements_of_type('osm') + for f in instance.xform.get_survey_elements_of_type("osm") ] osm_filenames = { - field: instance.json[field] - for field in fields if field in instance.json + field: instance.json[field] for field in fields if field in instance.json } for osm in osm_attachments: try: osm_xml = osm.media_file.read() if isinstance(osm_xml, bytes): - osm_xml = osm_xml.decode('utf-8') + osm_xml = osm_xml.decode("utf-8") except IOError as e: logging.exception("IOError saving osm data: %s" % str(e)) continue @@ -178,7 +172,7 @@ def save_osm_data(instance_id): filename = None field_name = None for k, v in osm_filenames.items(): - if osm.filename.startswith(v.replace('.osm', '')): + if osm.filename.startswith(v.replace(".osm", "")): filename = v field_name = k break @@ -193,26 +187,27 @@ def save_osm_data(instance_id): osm_data = OsmData( instance=instance, xml=osm_xml, - osm_id=osmd['osm_id'], - osm_type=osmd['osm_type'], - tags=osmd['tags'], - geom=GeometryCollection(osmd['geom']), + osm_id=osmd["osm_id"], + osm_type=osmd["osm_type"], + tags=osmd["tags"], + geom=GeometryCollection(osmd["geom"]), filename=filename, - field_name=field_name) + field_name=field_name, + ) osm_data.save() except IntegrityError: with transaction.atomic(): - osm_data = OsmData.objects.exclude( - xml=osm_xml).filter( - instance=instance, - field_name=field_name).first() + osm_data = ( + OsmData.objects.exclude(xml=osm_xml) + .filter(instance=instance, field_name=field_name) + .first() + ) if osm_data: osm_data.xml = osm_xml - osm_data.osm_id = osmd['osm_id'] - osm_data.osm_type = osmd['osm_type'] - osm_data.tags = osmd['tags'] - osm_data.geom = GeometryCollection( - osmd['geom']) + osm_data.osm_id = osmd["osm_id"] + osm_data.osm_type = osmd["osm_type"] + osm_data.tags = osmd["tags"] + osm_data.geom = GeometryCollection(osmd["geom"]) osm_data.filename = filename osm_data.save() instance.save() diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 5637eeff09..9507d44f25 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -593,3 +593,5 @@ def configure_logging(logger, **kwargs): XFORM_SUBMISSION_STAT_CACHE_TIME = 600 XFORM_CHARTS_CACHE_TIME = 600 + +SLAVE_DATABASES = [] diff --git a/requirements/base.in b/requirements/base.in index 24480fac29..81444c0dc4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,10 +2,10 @@ -e . # installed from Git -#-e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest -#-e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest -#-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router -#-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip -#-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient -#-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client -#-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest +-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router +-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip +-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient +-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client +-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index 0e9e81d503..f12ceab567 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -4,6 +4,20 @@ # # pip-compile --output-file=requirements/base.pip requirements/base.in # +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest + # via -r requirements/base.in +-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router + # via -r requirements/base.in +-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client + # via -r requirements/base.in +-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc + # via -r requirements/base.in +-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip + # via -r requirements/base.in +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest + # via -r requirements/base.in +-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient + # via -r requirements/base.in alabaster==0.7.12 # via sphinx amqp==5.1.1 @@ -16,18 +30,34 @@ asgiref==3.5.0 # via django async-timeout==4.0.2 # via redis +attrs==21.4.0 + # via + # jsonlines + # jsonschema babel==2.9.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery +boto3==1.21.43 + # via tabulator +botocore==1.24.43 + # via + # boto3 + # s3transfer +cached-property==1.5.2 + # via tableschema celery==5.2.6 # via onadata certifi==2021.10.8 # via requests cffi==1.15.0 # via cryptography +chardet==4.0.0 + # via + # datapackage + # tabulator charset-normalizer==2.0.12 # via requests click==8.1.2 @@ -36,6 +66,9 @@ click==8.1.2 # click-didyoumean # click-plugins # click-repl + # datapackage + # tableschema + # tabulator click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 @@ -46,6 +79,9 @@ cryptography==36.0.2 # via # jwcrypto # onadata + # pyjwt +datapackage==1.15.2 + # via pyfloip defusedxml==0.7.1 # via # djangorestframework-xml @@ -72,6 +108,7 @@ django==3.2.13 # djangorestframework # djangorestframework-guardian # djangorestframework-jsonapi + # ona-oidc # onadata django-activity-stream==1.4.0 # via onadata @@ -111,6 +148,7 @@ djangorestframework==3.13.1 # djangorestframework-gis # djangorestframework-guardian # djangorestframework-jsonapi + # ona-oidc # onadata djangorestframework-csv==2.1.1 # via onadata @@ -136,26 +174,50 @@ flake8==4.0.1 # via onadata fleming==0.7.0 # via django-query-builder +future==0.18.2 + # via python-json2xlsclient geojson==2.5.0 # via onadata +greenlet==1.1.2 + # via sqlalchemy httmock==1.4.0 # via onadata httplib2==0.20.4 - # via onadata + # via + # oauth2client + # onadata idna==3.3 # via requests +ijson==3.1.4 + # via tabulator imagesize==1.3.0 # via sphinx inflection==0.5.1 # via djangorestframework-jsonapi +isodate==0.6.1 + # via tableschema jinja2==3.1.1 # via sphinx +jmespath==1.0.0 + # via + # boto3 + # botocore +jsonlines==3.0.0 + # via tabulator jsonpickle==2.1.0 # via onadata +jsonpointer==2.3 + # via datapackage +jsonschema==4.4.0 + # via + # datapackage + # tableschema jwcrypto==1.0 # via django-oauth-toolkit kombu==5.2.4 # via celery +linear-tsv==1.1.0 + # via tabulator lxml==4.8.0 # via onadata markdown==3.3.6 @@ -180,6 +242,7 @@ openpyxl==3.0.9 # via # onadata # pyxform + # tabulator packaging==21.3 # via # redis @@ -192,6 +255,15 @@ pillow==9.1.0 # onadata prompt-toolkit==3.0.29 # via click-repl +psycopg2-binary==2.9.3 + # via onadata +pyasn1==0.4.8 + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via oauth2client pycodestyle==2.8.0 # via flake8 pycparser==2.21 @@ -200,8 +272,10 @@ pyflakes==2.4.0 # via flake8 pygments==2.11.2 # via sphinx -pyjwt==2.3.0 - # via onadata +pyjwt[crypto]==2.3.0 + # via + # ona-oidc + # onadata pylibmc==1.6.1 # via onadata pymongo==4.1.1 @@ -210,11 +284,15 @@ pyparsing==3.0.8 # via # httplib2 # packaging +pyrsistent==0.18.1 + # via jsonschema python-dateutil==2.8.2 # via # analytics-python + # botocore # fleming # onadata + # tableschema python-memcached==1.59 # via onadata pytz==2022.1 @@ -227,7 +305,9 @@ pytz==2022.1 # fleming # onadata pyxform==1.9.0 - # via onadata + # via + # onadata + # pyfloip raven==6.10.0 # via onadata recaptcha-client==1.0.6 @@ -237,13 +317,24 @@ redis==4.2.2 requests==2.27.1 # via # analytics-python + # datapackage # django-oauth-toolkit # httmock + # ona-oidc # onadata + # python-json2xlsclient # requests-mock # sphinx + # tableschema + # tabulator requests-mock==1.9.3 # via onadata +rfc3986==2.0.0 + # via tableschema +rsa==4.8 + # via oauth2client +s3transfer==0.5.2 + # via boto3 savreaderwriter==3.4.2 # via onadata simplejson==3.17.6 @@ -253,11 +344,17 @@ six==1.16.0 # analytics-python # appoptics-metrics # click-repl + # datapackage # django-query-builder # djangorestframework-csv + # isodate + # linear-tsv + # oauth2client # python-dateutil # python-memcached # requests-mock + # tableschema + # tabulator snowballstemmer==2.2.0 # via sphinx sphinx==4.5.0 @@ -274,16 +371,29 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +sqlalchemy==1.4.35 + # via tabulator sqlparse==0.4.2 # via # django # django-debug-toolbar +tableschema==1.20.2 + # via datapackage +tabulator==1.53.5 + # via + # datapackage + # tableschema unicodecsv==0.14.1 # via + # datapackage # djangorestframework-csv # onadata + # tableschema + # tabulator urllib3==1.26.9 - # via requests + # via + # botocore + # requests uwsgi==2.0.20 # via onadata vine==5.0.0 @@ -299,6 +409,7 @@ xlrd==2.0.1 # via # onadata # pyxform + # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 diff --git a/setup.cfg b/setup.cfg index 1b794f9ed5..398ac0baa8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,7 +55,7 @@ install_requires = #tagging django-taggit #database - #psycopg2-binary>2.7.1 + psycopg2-binary>2.7.1 pymongo #sms support dict2xml From 4f6ea563e9a848212cfb6f1b934453bbe6c24bef Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:22:15 +0300 Subject: [PATCH 004/234] django.utils.six - Remove usage of this vendored library or switch to six. --- .../commands/test_create_user_profiles.py | 17 +- .../management/commands/test_delete_users.py | 32 +- .../test_increase_odk_token_lifetime.py | 14 +- .../test_retrieve_org_or_project_list.py | 80 ++- .../apps/api/viewsets/briefcase_viewset.py | 172 +++--- onadata/apps/api/viewsets/data_viewset.py | 452 ++++++++------- onadata/apps/api/viewsets/xform_viewset.py | 4 +- .../tests/test_transfer_project_command.py | 129 +++-- onadata/apps/logger/views.py | 528 ++++++++++-------- onadata/libs/filters.py | 370 ++++++------ onadata/libs/renderers/renderers.py | 4 +- onadata/libs/utils/api_export_tools.py | 367 ++++++------ onadata/settings/common.py | 6 +- 13 files changed, 1143 insertions(+), 1032 deletions(-) diff --git a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py index 595d2226e0..e657b05999 100644 --- a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py +++ b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """Test create user profile management command.""" from django.contrib.auth.models import User -from onadata.apps.main.models.user_profile import UserProfile from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO + +from onadata.apps.main.models.user_profile import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -16,19 +17,13 @@ def test_create_user_profiles(self): successfully creates a user profile for users missing profiles. """ - user = User.objects.create( - username='dave', email='dave@example.com') + user = User.objects.create(username="dave", email="dave@example.com") with self.assertRaises(UserProfile.DoesNotExist): _ = user.profile out = StringIO() - call_command( - 'create_user_profiles', - stdout=out - ) + call_command("create_user_profiles", stdout=out) user.refresh_from_db() # Assert profile is retrievable; profile = user.profile self.assertEqual(profile.user, user) - self.assertEqual( - 'User Profiles successfully created.\n', - out.getvalue()) + self.assertEqual("User Profiles successfully created.\n", out.getvalue()) diff --git a/onadata/apps/api/tests/management/commands/test_delete_users.py b/onadata/apps/api/tests/management/commands/test_delete_users.py index 329e7eac89..ce975e13f5 100644 --- a/onadata/apps/api/tests/management/commands/test_delete_users.py +++ b/onadata/apps/api/tests/management/commands/test_delete_users.py @@ -3,59 +3,51 @@ """ import sys from unittest import mock -from django.utils.six import StringIO +from six import StringIO from django.contrib.auth.models import User from django.core.management import call_command from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.api.management.commands.delete_users import \ - get_user_object_stats +from onadata.apps.api.management.commands.delete_users import get_user_object_stats class DeleteUserTest(TestBase): """ Test delete user management command. """ + def test_delete_users_with_input(self): """ Test that a user account is deleted automatically when the user_input field is provided as true """ - user = User.objects.create( - username="bruce", - email="bruce@gmail.com") + user = User.objects.create(username="bruce", email="bruce@gmail.com") username = user.username email = user.email out = StringIO() sys.stdout = out - new_user_details = [username+':'+email] + new_user_details = [username + ":" + email] call_command( - 'delete_users', - user_details=new_user_details, - user_input='True', - stdout=out) + "delete_users", user_details=new_user_details, user_input="True", stdout=out + ) - self.assertEqual( - "User bruce deleted successfully.", - out.getvalue()) + self.assertEqual("User bruce deleted successfully.", out.getvalue()) with self.assertRaises(User.DoesNotExist): User.objects.get(email="bruce@gmail.com") - @mock.patch( - "onadata.apps.api.management.commands.delete_users.input") + @mock.patch("onadata.apps.api.management.commands.delete_users.input") def test_delete_users_no_input(self, mock_input): # pylint: disable=R0201 """ Test that when user_input is not provided, the user account stats are provided for that user account before deletion """ - user = User.objects.create( - username="barbie", - email="barbie@gmail.com") + user = User.objects.create(username="barbie", email="barbie@gmail.com") username = user.username get_user_object_stats(username) mock_input.assert_called_with( "User account 'barbie' has 0 projects, " "0 forms and 0 submissions. " - "Do you wish to continue deleting this account?") + "Do you wish to continue deleting this account?" + ) diff --git a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py index 3cd040d68a..af54161d35 100644 --- a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py +++ b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.models.odk_token import ODKToken @@ -10,21 +10,17 @@ class IncreaseODKTokenLifetimeTest(TestBase): def test_increase_odk_token_lifetime(self): - user = User.objects.create( - username='dave', email='dave@example.com') + user = User.objects.create(username="dave", email="dave@example.com") token = ODKToken.objects.create(user=user) expiry_date = token.expires out = StringIO() call_command( - 'increase_odk_token_lifetime', - days=2, - username=user.username, - stdout=out + "increase_odk_token_lifetime", days=2, username=user.username, stdout=out ) self.assertEqual( - 'Increased the lifetime of ODK Token for user dave\n', - out.getvalue()) + "Increased the lifetime of ODK Token for user dave\n", out.getvalue() + ) token.refresh_from_db() self.assertEqual(expiry_date + timedelta(days=2), token.expires) diff --git a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py index 1fe1a383e7..703f838dfe 100644 --- a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py +++ b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py @@ -1,11 +1,10 @@ import json + from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.libs.serializers.share_project_serializer import \ - ShareProjectSerializer +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.libs.serializers.share_project_serializer import ShareProjectSerializer class TestRetrieveOrgOrProjectListCommand(TestAbstractViewSet): @@ -13,52 +12,48 @@ def test_retrieve_org_or_project_list(self): self._org_create() self._project_create() - user = self._create_user_profile( - {'username': 'alice'}).user + user = self._create_user_profile({"username": "alice"}).user share_data = { - 'project': self.project.id, - 'username': user.username, - 'role': 'editor' + "project": self.project.id, + "username": user.username, + "role": "editor", } serializer = ShareProjectSerializer(data=share_data) self.assertTrue(serializer.is_valid()) serializer.save() out = StringIO() - call_command( - 'retrieve_org_or_project_list', - stdout=out - ) + call_command("retrieve_org_or_project_list", stdout=out) expected_project_data = { self.project.name: { user.username: { - 'first_name': user.first_name, - 'last_name': user.last_name, - 'is_org': False, - 'role': 'editor' + "first_name": user.first_name, + "last_name": user.last_name, + "is_org": False, + "role": "editor", }, self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'is_org': False, - 'role': 'owner' - } + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "is_org": False, + "role": "owner", + }, } } expected_org_data = { self.organization.name: { self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'role': 'owner' + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "role": "owner", }, self.organization.user.username: { - 'first_name': self.organization.user.first_name, - 'last_name': self.organization.user.last_name, - 'role': 'owner' - } + "first_name": self.organization.user.first_name, + "last_name": self.organization.user.last_name, + "role": "owner", + }, } } @@ -66,31 +61,20 @@ def test_retrieve_org_or_project_list(self): expected_data.update(expected_project_data) expected_data.update(expected_org_data) - self.assertEqual( - expected_data, - json.loads(out.getvalue()) - ) + self.assertEqual(expected_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - project_ids=f'{self.project.id}', - stdout=out - ) - self.assertEqual( - expected_project_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", project_ids=f"{self.project.id}", stdout=out ) + self.assertEqual(expected_project_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - organization_ids=f'{self.organization.id}', - stdout=out - ) - self.assertEqual( - expected_org_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", + organization_ids=f"{self.organization.id}", + stdout=out, ) + self.assertEqual(expected_org_data, json.loads(out.getvalue())) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 79522e821a..cb64b0f4b8 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -1,11 +1,13 @@ +import six + from xml.dom import NotFoundErr + from django.conf import settings from django.core.files import File from django.core.validators import ValidationError from django.contrib.auth.models import User from django.http import Http404 from django.utils.translation import ugettext as _ -from django.utils import six from rest_framework import exceptions from rest_framework import mixins @@ -38,13 +40,13 @@ def _extract_uuid(text): if isinstance(text, six.string_types): - form_id_parts = text.split('/') + form_id_parts = text.split("/") if form_id_parts.__len__() < 2: - raise ValidationError(_(u"Invalid formId %s." % text)) + raise ValidationError(_("Invalid formId %s." % text)) text = form_id_parts[1] - text = text[text.find("@key="):-1].replace("@key=", "") + text = text[text.find("@key=") : -1].replace("@key=", "") if text.startswith("uuid:"): text = text.replace("uuid:", "") @@ -54,7 +56,7 @@ def _extract_uuid(text): def _extract_id_string(formId): if isinstance(formId, six.string_types): - return formId[0:formId.find('[')] + return formId[0 : formId.find("[")] return formId @@ -66,45 +68,49 @@ def _parse_int(num): pass -class BriefcaseViewset(mixins.CreateModelMixin, - mixins.RetrieveModelMixin, mixins.ListModelMixin, - viewsets.GenericViewSet): +class BriefcaseViewset( + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): """ Implements the [Briefcase Aggregate API](\ https://code.google.com/p/opendatakit/wiki/BriefcaseAggregateAPI). """ + authentication_classes = (DigestAuthentication,) filter_backends = (filters.AnonDjangoObjectPermissionFilter,) queryset = XForm.objects.all() - permission_classes = (permissions.IsAuthenticated, - ViewDjangoObjectPermissions) + permission_classes = (permissions.IsAuthenticated, ViewDjangoObjectPermissions) renderer_classes = (TemplateXMLRenderer, BrowsableAPIRenderer) serializer_class = XFormListSerializer - template_name = 'openrosa_response.xml' + template_name = "openrosa_response.xml" def get_object(self, queryset=None): - formId = self.request.GET.get('formId', '') + formId = self.request.GET.get("formId", "") id_string = _extract_id_string(formId) uuid = _extract_uuid(formId) - username = self.kwargs.get('username') + username = self.kwargs.get("username") - obj = get_object_or_404(Instance, - xform__user__username__iexact=username, - xform__id_string__iexact=id_string, - uuid=uuid) + obj = get_object_or_404( + Instance, + xform__user__username__iexact=username, + xform__id_string__iexact=id_string, + uuid=uuid, + ) self.check_object_permissions(self.request, obj.xform) return obj def filter_queryset(self, queryset): - username = self.kwargs.get('username') + username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication self.permission_denied(self.request) if username is not None and self.request.user.is_anonymous: - profile = get_object_or_404( - UserProfile, user__username__iexact=username) + profile = get_object_or_404(UserProfile, user__username__iexact=username) if profile.require_auth: # raises a permission denied exception, forces authentication @@ -114,27 +120,24 @@ def filter_queryset(self, queryset): else: queryset = super(BriefcaseViewset, self).filter_queryset(queryset) - formId = self.request.GET.get('formId', '') + formId = self.request.GET.get("formId", "") - if formId.find('[') != -1: + if formId.find("[") != -1: formId = _extract_id_string(formId) - xform_kwargs = { - 'queryset': queryset, - 'id_string__iexact': formId - } + xform_kwargs = {"queryset": queryset, "id_string__iexact": formId} if username: - xform_kwargs['user__username__iexact'] = username + xform_kwargs["user__username__iexact"] = username xform = get_form(xform_kwargs) self.check_object_permissions(self.request, xform) instances = Instance.objects.filter( - xform=xform, deleted_at__isnull=True).values( - 'pk', 'uuid') + xform=xform, deleted_at__isnull=True + ).values("pk", "uuid") if xform.encrypted: instances = instances.filter(media_all_received=True) - instances = instances.order_by('pk') - num_entries = self.request.GET.get('numEntries') - cursor = self.request.GET.get('cursor') + instances = instances.order_by("pk") + num_entries = self.request.GET.get("numEntries") + cursor = self.request.GET.get("cursor") cursor = _parse_int(cursor) if cursor: @@ -152,7 +155,7 @@ def filter_queryset(self, queryset): if instance_count > 0: last_instance = instances[instance_count - 1] - self.resumptionCursor = last_instance.get('pk') + self.resumptionCursor = last_instance.get("pk") elif instance_count == 0 and cursor: self.resumptionCursor = cursor else: @@ -161,23 +164,28 @@ def filter_queryset(self, queryset): return instances def create(self, request, *args, **kwargs): - if request.method.upper() == 'HEAD': - return Response(status=status.HTTP_204_NO_CONTENT, - headers=get_openrosa_headers(request), - template_name=self.template_name) - - xform_def = request.FILES.get('form_def_file', None) + if request.method.upper() == "HEAD": + return Response( + status=status.HTTP_204_NO_CONTENT, + headers=get_openrosa_headers(request), + template_name=self.template_name, + ) + + xform_def = request.FILES.get("form_def_file", None) response_status = status.HTTP_201_CREATED - username = kwargs.get('username') - form_user = (username and get_object_or_404(User, username=username)) \ - or request.user + username = kwargs.get("username") + form_user = ( + username and get_object_or_404(User, username=username) + ) or request.user - if not request.user.has_perm('can_add_xform', form_user.profile): + if not request.user.has_perm("can_add_xform", form_user.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % - {'user': request.user.username, - 'account': form_user.username})) + detail=_( + "User %(user)s has no permission to add xforms to " + "account %(account)s" + % {"user": request.user.username, "account": form_user.username} + ) + ) data = {} if isinstance(xform_def, File): @@ -185,30 +193,34 @@ def create(self, request, *args, **kwargs): dd = publish_form(do_form_upload.publish_xform) if isinstance(dd, XForm): - data['message'] = _( - u"%s successfully published." % dd.id_string) + data["message"] = _("%s successfully published." % dd.id_string) else: - data['message'] = dd['text'] + data["message"] = dd["text"] response_status = status.HTTP_400_BAD_REQUEST else: - data['message'] = _(u"Missing xml file.") + data["message"] = _("Missing xml file.") response_status = status.HTTP_400_BAD_REQUEST - return Response(data, status=response_status, - headers=get_openrosa_headers(request, - location=False), - template_name=self.template_name) + return Response( + data, + status=response_status, + headers=get_openrosa_headers(request, location=False), + template_name=self.template_name, + ) def list(self, request, *args, **kwargs): self.object_list = self.filter_queryset(self.get_queryset()) - data = {'instances': self.object_list, - 'resumptionCursor': self.resumptionCursor} + data = { + "instances": self.object_list, + "resumptionCursor": self.resumptionCursor, + } - return Response(data, - headers=get_openrosa_headers(request, - location=False), - template_name='submissionList.xml') + return Response( + data, + headers=get_openrosa_headers(request, location=False), + template_name="submissionList.xml", + ) def retrieve(self, request, *args, **kwargs): self.object = self.get_object() @@ -216,52 +228,54 @@ def retrieve(self, request, *args, **kwargs): xml_obj = clean_and_parse_xml(self.object.xml) submission_xml_root_node = xml_obj.documentElement submission_xml_root_node.setAttribute( - 'instanceID', u'uuid:%s' % self.object.uuid) + "instanceID", "uuid:%s" % self.object.uuid + ) submission_xml_root_node.setAttribute( - 'submissionDate', self.object.date_created.isoformat() + "submissionDate", self.object.date_created.isoformat() ) if getattr(settings, "SUPPORT_BRIEFCASE_SUBMISSION_DATE", True): # Remove namespace attribute if any try: - submission_xml_root_node.removeAttribute('xmlns') + submission_xml_root_node.removeAttribute("xmlns") except NotFoundErr: pass data = { - 'submission_data': submission_xml_root_node.toxml(), - 'media_files': Attachment.objects.filter(instance=self.object), - 'host': request.build_absolute_uri().replace( - request.get_full_path(), '') + "submission_data": submission_xml_root_node.toxml(), + "media_files": Attachment.objects.filter(instance=self.object), + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), } return Response( data, headers=get_openrosa_headers(request, location=False), - template_name='downloadSubmission.xml' + template_name="downloadSubmission.xml", ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def manifest(self, request, *args, **kwargs): self.object = self.get_object() - object_list = MetaData.objects.filter(data_type='media', - object_id=self.object.id) + object_list = MetaData.objects.filter( + data_type="media", object_id=self.object.id + ) context = self.get_serializer_context() - serializer = XFormManifestSerializer(object_list, many=True, - context=context) + serializer = XFormManifestSerializer(object_list, many=True, context=context) - return Response(serializer.data, - headers=get_openrosa_headers(request, location=False)) + return Response( + serializer.data, headers=get_openrosa_headers(request, location=False) + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def media(self, request, *args, **kwargs): self.object = self.get_object() - pk = kwargs.get('metadata') + pk = kwargs.get("metadata") if not pk: raise Http404() meta_obj = get_object_or_404( - MetaData, data_type='media', xform=self.object, pk=pk) + MetaData, data_type="media", xform=self.object, pk=pk + ) return get_media_file_response(meta_obj) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 918b7a6980..47b205a123 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -3,6 +3,8 @@ from builtins import str as text from typing import Union +import six + from django.conf import settings from django.core.exceptions import PermissionDenied from django.db.models import Q @@ -10,7 +12,6 @@ from django.db.utils import DataError, OperationalError from django.http import Http404 from django.http import StreamingHttpResponse -from django.utils import six from django.utils import timezone from distutils.util import strtobool from django.utils.translation import ugettext as _ @@ -28,9 +29,7 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models import OsmData, MergedXForm from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.instance import ( - Instance, - FormInactiveError) +from onadata.apps.logger.models.instance import Instance, FormInactiveError from onadata.apps.logger.models.xform import XForm from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED from onadata.apps.messaging.serializers import send_message @@ -43,18 +42,23 @@ from onadata.libs.exceptions import EnketoError from onadata.libs.exceptions import NoRecordsPermission from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import CountOverridablePageNumberPagination -from onadata.libs.permissions import CAN_DELETE_SUBMISSION, \ - filter_queryset_xform_meta_perms, filter_queryset_xform_meta_perms_sql +from onadata.libs.permissions import ( + CAN_DELETE_SUBMISSION, + filter_queryset_xform_meta_perms, + filter_queryset_xform_meta_perms_sql, +) from onadata.libs.renderers import renderers from onadata.libs.serializers.data_serializer import ( - DataInstanceSerializer, DataInstanceXMLSerializer, - InstanceHistorySerializer) + DataInstanceSerializer, + DataInstanceXMLSerializer, + InstanceHistorySerializer, +) from onadata.libs.serializers.data_serializer import DataSerializer from onadata.libs.serializers.data_serializer import JsonDataSerializer from onadata.libs.serializers.data_serializer import OSMSerializer @@ -63,20 +67,20 @@ from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.viewer_tools import get_form_url, get_enketo_urls -SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) BaseViewset = get_baseviewset_class() def get_data_and_form(kwargs): - data_id = text(kwargs.get('dataid')) + data_id = text(kwargs.get("dataid")) if not data_id.isdigit(): - raise ParseError(_(u"Data ID should be an integer")) + raise ParseError(_("Data ID should be an integer")) - return (data_id, kwargs.get('format')) + return (data_id, kwargs.get("format")) def delete_instance(instance, user): @@ -93,11 +97,14 @@ def delete_instance(instance, user): raise ParseError(text(e)) -class DataViewSet(AnonymousUserPublicFormsMixin, - AuthenticateHeaderMixin, - ETagsMixin, CacheControlMixin, - BaseViewset, - ModelViewSet): +class DataViewSet( + AnonymousUserPublicFormsMixin, + AuthenticateHeaderMixin, + ETagsMixin, + CacheControlMixin, + BaseViewset, + ModelViewSet, +): """ This endpoint provides access to submitted data. """ @@ -116,16 +123,18 @@ class DataViewSet(AnonymousUserPublicFormsMixin, renderers.FLOIPRenderer, ] - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.XFormOwnerFilter, - filters.DataFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.XFormOwnerFilter, + filters.DataFilter, + ) serializer_class = DataSerializer permission_classes = (XFormPermissions,) - lookup_field = 'pk' - lookup_fields = ('pk', 'dataid') + lookup_field = "pk" + lookup_fields = ("pk", "dataid") extra_lookup_fields = None data_count = None - public_data_endpoint = 'public' + public_data_endpoint = "public" pagination_class = CountOverridablePageNumberPagination queryset = XForm.objects.filter(deleted_at__isnull=True) @@ -134,24 +143,22 @@ def get_serializer_class(self): pk_lookup, dataid_lookup = self.lookup_fields pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - fmt = self.kwargs.get('format', self.request.GET.get("format")) + fmt = self.kwargs.get("format", self.request.GET.get("format")) sort = self.request.GET.get("sort") fields = self.request.GET.get("fields") if fmt == Attachment.OSM: serializer_class = OSMSerializer - elif fmt == 'geojson': + elif fmt == "geojson": serializer_class = GeoJsonSerializer - elif fmt == 'xml': + elif fmt == "xml": serializer_class = DataInstanceXMLSerializer - elif pk is not None and dataid is None \ - and pk != self.public_data_endpoint: + elif pk is not None and dataid is None and pk != self.public_data_endpoint: if sort or fields: serializer_class = JsonDataSerializer else: serializer_class = DataInstanceSerializer else: - serializer_class = \ - super(DataViewSet, self).get_serializer_class() + serializer_class = super(DataViewSet, self).get_serializer_class() return serializer_class @@ -165,42 +172,42 @@ def get_object(self, queryset=None): try: int(dataid) except ValueError: - raise ParseError(_(u"Invalid dataid %(dataid)s" - % {'dataid': dataid})) + raise ParseError(_("Invalid dataid %(dataid)s" % {"dataid": dataid})) if not obj.is_merged_dataset: - obj = get_object_or_404(Instance, pk=dataid, xform__pk=pk, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform__pk=pk, deleted_at__isnull=True + ) else: xforms = obj.mergedxform.xforms.filter(deleted_at__isnull=True) - pks = [xform_id - for xform_id in xforms.values_list('pk', flat=True)] + pks = [xform_id for xform_id in xforms.values_list("pk", flat=True)] - obj = get_object_or_404(Instance, pk=dataid, xform_id__in=pks, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform_id__in=pks, deleted_at__isnull=True + ) return obj def _get_public_forms_queryset(self): - return XForm.objects.filter(Q(shared=True) | Q(shared_data=True), - deleted_at__isnull=True) + return XForm.objects.filter( + Q(shared=True) | Q(shared_data=True), deleted_at__isnull=True + ) def _filtered_or_shared_qs(self, qs, pk): filter_kwargs = {self.lookup_field: pk} - qs = qs.filter(**filter_kwargs).only('id', 'shared') + qs = qs.filter(**filter_kwargs).only("id", "shared") if not qs: - filter_kwargs['shared_data'] = True - qs = XForm.objects.filter(**filter_kwargs).only('id', 'shared') + filter_kwargs["shared_data"] = True + qs = XForm.objects.filter(**filter_kwargs).only("id", "shared") if not qs: - raise Http404(_(u"No data matches with given query.")) + raise Http404(_("No data matches with given query.")) return qs def filter_queryset(self, queryset, view=None): - qs = super(DataViewSet, self).filter_queryset( - queryset.only('id', 'shared')) + qs = super(DataViewSet, self).filter_queryset(queryset.only("id", "shared")) pk = self.kwargs.get(self.lookup_field) if pk: @@ -210,73 +217,81 @@ def filter_queryset(self, queryset, view=None): if pk == self.public_data_endpoint: qs = self._get_public_forms_queryset() else: - raise ParseError(_(u"Invalid pk %(pk)s" % {'pk': pk})) + raise ParseError(_("Invalid pk %(pk)s" % {"pk": pk})) else: qs = self._filtered_or_shared_qs(qs, pk) else: - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") qs = qs.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') + not_tagged = not_tagged.split(",") qs = qs.exclude(tags__name__in=not_tagged) return qs - @action(methods=['GET', 'POST', 'DELETE'], detail=True, - extra_lookup_fields=['label', ]) + @action( + methods=["GET", "POST", "DELETE"], + detail=True, + extra_lookup_fields=[ + "label", + ], + ) def labels(self, request, *args, **kwargs): http_status = status.HTTP_400_BAD_REQUEST self.object = instance = self.get_object() - if request.method == 'POST': + if request.method == "POST": add_tags_to_instance(request, instance) http_status = status.HTTP_201_CREATED tags = instance.tags - label = kwargs.get('label') + label = kwargs.get("label") - if request.method == 'GET' and label: - data = [tag['name'] for tag in - tags.filter(name=label).values('name')] + if request.method == "GET" and label: + data = [tag["name"] for tag in tags.filter(name=label).values("name")] - elif request.method == 'DELETE' and label: + elif request.method == "DELETE" and label: count = tags.count() tags.remove(label) # Accepted, label does not exist hence nothing removed - http_status = status.HTTP_200_OK if count > tags.count() \ + http_status = ( + status.HTTP_200_OK + if count > tags.count() else status.HTTP_404_NOT_FOUND + ) data = list(tags.names()) else: data = list(tags.names()) - if request.method == 'GET': + if request.method == "GET": http_status = status.HTTP_200_OK self.etag_data = data return Response(data, status=http_status) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def enketo(self, request, *args, **kwargs): self.object = self.get_object() data = {} if isinstance(self.object, XForm): - raise ParseError(_(u"Data id not provided.")) - elif(isinstance(self.object, Instance)): + raise ParseError(_("Data id not provided.")) + elif isinstance(self.object, Instance): if request.user.has_perm("change_xform", self.object.xform): - return_url = request.query_params.get('return_url') + return_url = request.query_params.get("return_url") form_url = get_form_url( request, self.object.xform.user.username, - xform_pk=self.object.xform.id) + xform_pk=self.object.xform.id, + ) if not return_url: - raise ParseError(_(u"return_url not provided.")) + raise ParseError(_("return_url not provided.")) try: data = get_enketo_urls( @@ -284,52 +299,52 @@ def enketo(self, request, *args, **kwargs): self.object.xform.id_string, instance_id=self.object.uuid, instance_xml=self.object.xml, - return_url=return_url) + return_url=return_url, + ) if "edit_url" in data: data["url"] = data.pop("edit_url") except EnketoError as e: raise ParseError(text(e)) else: - raise PermissionDenied(_(u"You do not have edit permissions.")) + raise PermissionDenied(_("You do not have edit permissions.")) self.etag_data = data return Response(data=data) def destroy(self, request, *args, **kwargs): - instance_ids = request.data.get('instance_ids') - delete_all_submissions = strtobool( - request.data.get('delete_all', 'False')) + instance_ids = request.data.get("instance_ids") + delete_all_submissions = strtobool(request.data.get("delete_all", "False")) self.object = self.get_object() if isinstance(self.object, XForm): if not instance_ids and not delete_all_submissions: - raise ParseError(_(u"Data id(s) not provided.")) + raise ParseError(_("Data id(s) not provided.")) else: initial_count = self.object.submission_count() if delete_all_submissions: # Update timestamp only for active records - self.object.instances.filter( - deleted_at__isnull=True).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user) + self.object.instances.filter(deleted_at__isnull=True).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, + ) else: - instance_ids = [ - x for x in instance_ids.split(',') if x.isdigit()] + instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] if not instance_ids: - raise ParseError(_(u"Invalid data ids were provided.")) + raise ParseError(_("Invalid data ids were provided.")) self.object.instances.filter( id__in=instance_ids, xform=self.object, # do not update this timestamp when the record have # already been deleted. - deleted_at__isnull=True + deleted_at__isnull=True, ).update( deleted_at=timezone.now(), date_modified=timezone.now(), - deleted_by=request.user) + deleted_by=request.user, + ) # updates the num_of_submissions for the form. after_count = self.object.submission_count(force_update=True) @@ -337,37 +352,39 @@ def destroy(self, request, *args, **kwargs): # update the date modified field of the project self.object.project.date_modified = timezone.now() - self.object.project.save(update_fields=['date_modified']) + self.object.project.save(update_fields=["date_modified"]) # send message send_message( - instance_id=instance_ids, target_id=self.object.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) + instance_id=instance_ids, + target_id=self.object.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) return Response( data={ - "message": - "%d records were deleted" % - number_of_records_deleted + "message": "%d records were deleted" % number_of_records_deleted }, - status=status.HTTP_200_OK + status=status.HTTP_200_OK, ) elif isinstance(self.object, Instance): - if request.user.has_perm( - CAN_DELETE_SUBMISSION, self.object.xform): + if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): instance_id = self.object.pk delete_instance(self.object, request.user) # send message send_message( - instance_id=instance_id, target_id=self.object.xform.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) + instance_id=instance_id, + target_id=self.object.xform.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) else: - raise PermissionDenied(_(u"You do not have delete " - u"permissions.")) + raise PermissionDenied(_("You do not have delete " "permissions.")) return Response(status=status.HTTP_204_NO_CONTENT) @@ -375,50 +392,56 @@ def retrieve(self, request, *args, **kwargs): data_id, _format = get_data_and_form(kwargs) self.object = instance = self.get_object() - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": return Response(instance.json) - elif _format == 'xml': + elif _format == "xml": return Response(instance.xml) - elif _format == 'geojson': - return super(DataViewSet, self)\ - .retrieve(request, *args, **kwargs) + elif _format == "geojson": + return super(DataViewSet, self).retrieve(request, *args, **kwargs) elif _format == Attachment.OSM: serializer = self.get_serializer(instance.osm_data.all()) return Response(serializer.data) else: raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) + _( + "'%(_format)s' format unknown or not implemented!" + % {"_format": _format} + ) + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def history(self, request, *args, **kwargs): data_id, _format = get_data_and_form(kwargs) instance = self.get_object() # retrieve all history objects and return them - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": instance_history = instance.submission_history.all() - serializer = InstanceHistorySerializer( - instance_history, many=True) + serializer = InstanceHistorySerializer(instance_history, many=True) return Response(serializer.data) else: raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) + _( + "'%(_format)s' format unknown or not implemented!" + % {"_format": _format} + ) + ) def _set_pagination_headers( - self, xform: XForm, current_page: Union[int, str], - current_page_size: Union[int, str] = - SUBMISSION_RETRIEVAL_THRESHOLD): + self, + xform: XForm, + current_page: Union[int, str], + current_page_size: Union[int, str] = SUBMISSION_RETRIEVAL_THRESHOLD, + ): """ Sets the self.headers value for the viewset """ import math url = self.request.build_absolute_uri() - query = self.request.query_params.get('query') - base_url = url.split('?')[0] + query = self.request.query_params.get("query") + base_url = url.split("?")[0] if query: num_of_records = self.object_list.count() else: @@ -443,36 +466,33 @@ def _set_pagination_headers( if (current_page * current_page_size) < num_of_records: next_page_url = ( - f"{base_url}?page={current_page + 1}&" - f"page_size={current_page_size}") + f"{base_url}?page={current_page + 1}&" f"page_size={current_page_size}" + ) if current_page > 1: prev_page_url = ( - f"{base_url}?page={current_page - 1}" - f"&page_size={current_page_size}") + f"{base_url}?page={current_page - 1}" f"&page_size={current_page_size}" + ) last_page = math.ceil(num_of_records / current_page_size) if last_page != current_page and last_page != current_page + 1: - last_page_url = ( - f"{base_url}?page={last_page}&page_size={current_page_size}" - ) + last_page_url = f"{base_url}?page={last_page}&page_size={current_page_size}" if current_page != 1: - first_page_url = ( - f"{base_url}?page=1&page_size={current_page_size}" - ) + first_page_url = f"{base_url}?page=1&page_size={current_page_size}" - if not hasattr(self, 'headers'): + if not hasattr(self, "headers"): self.headers = {} for rel, link in ( - ('prev', prev_page_url), - ('next', next_page_url), - ('last', last_page_url), - ('first', first_page_url)): + ("prev", prev_page_url), + ("next", next_page_url), + ("last", last_page_url), + ("first", first_page_url), + ): if link: links.append(f'<{link}>; rel="{rel}"') - self.headers.update({'Link': ', '.join(links)}) + self.headers.update({"Link": ", ".join(links)}) def list(self, request, *args, **kwargs): fields = request.GET.get("fields") @@ -480,7 +500,7 @@ def list(self, request, *args, **kwargs): sort = request.GET.get("sort") start = parse_int(request.GET.get("start")) limit = parse_int(request.GET.get("limit")) - export_type = kwargs.get('format', request.GET.get("format")) + export_type = kwargs.get("format", request.GET.get("format")) lookup_field = self.lookup_field lookup = self.kwargs.get(lookup_field) is_public_request = lookup == self.public_data_endpoint @@ -494,117 +514,116 @@ def list(self, request, *args, **kwargs): if is_public_request: self.object_list = self._get_public_forms_queryset() elif lookup: - qs = self.filter_queryset( - self.get_queryset() - ).values_list('pk', 'is_merged_dataset') + qs = self.filter_queryset(self.get_queryset()).values_list( + "pk", "is_merged_dataset" + ) xform_id, is_merged_dataset = qs[0] if qs else (lookup, False) pks = [xform_id] if is_merged_dataset: merged_form = MergedXForm.objects.get(pk=xform_id) - qs = merged_form.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', 'num_of_submissions') + qs = merged_form.xforms.filter(deleted_at__isnull=True).values_list( + "id", "num_of_submissions" + ) try: - pks, num_of_submissions = [ - list(value) for value in zip(*qs)] + pks, num_of_submissions = [list(value) for value in zip(*qs)] num_of_submissions = sum(num_of_submissions) except ValueError: pks, num_of_submissions = [], 0 else: - num_of_submissions = XForm.objects.get( - id=xform_id).num_of_submissions + num_of_submissions = XForm.objects.get(id=xform_id).num_of_submissions self.object_list = Instance.objects.filter( - xform_id__in=pks, deleted_at=None).only('json') + xform_id__in=pks, deleted_at=None + ).only("json") # Enable ordering for XForms with Submissions that are less # than the SUBMISSION_RETRIEVAL_THRESHOLD if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD: - self.object_list = self.object_list.order_by('id') + self.object_list = self.object_list.order_by("id") xform = self.get_object() - self.object_list = \ - filter_queryset_xform_meta_perms(xform, request.user, - self.object_list) - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + self.object_list = filter_queryset_xform_meta_perms( + xform, request.user, self.object_list + ) + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") self.object_list = filters.InstanceFilter( - self.request.query_params, - queryset=self.object_list, - request=request + self.request.query_params, queryset=self.object_list, request=request ).qs if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") self.object_list = self.object_list.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') - self.object_list = \ - self.object_list.exclude(tags__name__in=not_tagged) + not_tagged = not_tagged.split(",") + self.object_list = self.object_list.exclude(tags__name__in=not_tagged) if ( - export_type is None or - export_type in ['json', 'jsonp', 'debug', 'xml']) \ - and hasattr(self, 'object_list'): - return self._get_data(query, fields, sort, start, limit, - is_public_request) + export_type is None or export_type in ["json", "jsonp", "debug", "xml"] + ) and hasattr(self, "object_list"): + return self._get_data(query, fields, sort, start, limit, is_public_request) xform = self.get_object() - kwargs = {'instance__xform': xform} + kwargs = {"instance__xform": xform} if export_type == Attachment.OSM: if request.GET: self.set_object_list( - query, fields, sort, start, limit, is_public_request) - kwargs = {'instance__in': self.object_list} - osm_list = OsmData.objects.filter(**kwargs).order_by('instance') + query, fields, sort, start, limit, is_public_request + ) + kwargs = {"instance__in": self.object_list} + osm_list = OsmData.objects.filter(**kwargs).order_by("instance") page = self.paginate_queryset(osm_list) serializer = self.get_serializer(page) return Response(serializer.data) - elif export_type is None or export_type in ['json']: + elif export_type is None or export_type in ["json"]: # perform default viewset retrieve, no data export return super(DataViewSet, self).list(request, *args, **kwargs) - elif export_type == 'geojson': + elif export_type == "geojson": serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) return custom_response_handler(request, xform, query, export_type) - def set_object_list( - self, query, fields, sort, start, limit, is_public_request): + def set_object_list(self, query, fields, sort, start, limit, is_public_request): try: enable_etag = True if not is_public_request: xform = self.get_object() self.data_count = xform.num_of_submissions - enable_etag = self.data_count <\ - SUBMISSION_RETRIEVAL_THRESHOLD + enable_etag = self.data_count < SUBMISSION_RETRIEVAL_THRESHOLD where, where_params = get_where_clause(query) if where: - self.object_list = self.object_list.extra(where=where, - params=where_params) + self.object_list = self.object_list.extra( + where=where, params=where_params + ) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( - self.get_object(), self.request.user, self.object_list) + self.get_object(), self.request.user, self.object_list + ) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: - query = \ - filter_queryset_xform_meta_perms_sql(self.get_object(), - self.request.user, - query) + query = filter_queryset_xform_meta_perms_sql( + self.get_object(), self.request.user, query + ) self.object_list = query_data( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields, - json_only=not self.kwargs.get('format') == 'xml') + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, + json_only=not self.kwargs.get("format") == "xml", + ) except NoRecordsPermission: self.object_list = [] @@ -615,11 +634,14 @@ def set_object_list( self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, ) - self.etag_hash = get_etag_hash_from_query( - records, sql, params) + self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: @@ -628,19 +650,18 @@ def set_object_list( def paginate_queryset(self, queryset): if self.paginator is None: return None - return self.paginator.paginate_queryset(queryset, - self.request, - view=self, - count=self.data_count) + return self.paginator.paginate_queryset( + queryset, self.request, view=self, count=self.data_count + ) def _get_data(self, query, fields, sort, start, limit, is_public_request): - self.set_object_list( - query, fields, sort, start, limit, is_public_request) + self.set_object_list(query, fields, sort, start, limit, is_public_request) - retrieval_threshold = getattr( - settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) - pagination_keys = [self.paginator.page_query_param, - self.paginator.page_size_query_param] + retrieval_threshold = getattr(settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) + pagination_keys = [ + self.paginator.page_query_param, + self.paginator.page_size_query_param, + ] query_param_keys = self.request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) @@ -653,13 +674,11 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): if should_paginate: self.paginator.page_size = retrieval_threshold - if not isinstance(self.object_list, types.GeneratorType) and \ - should_paginate: - current_page = query_param_keys.get( - self.paginator.page_query_param, 1) + if not isinstance(self.object_list, types.GeneratorType) and should_paginate: + current_page = query_param_keys.get(self.paginator.page_query_param, 1) current_page_size = query_param_keys.get( - self.paginator.page_size_query_param, - retrieval_threshold) + self.paginator.page_size_query_param, retrieval_threshold + ) self._set_pagination_headers( self.get_object(), @@ -672,7 +691,7 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): except OperationalError: self.object_list = self.paginate_queryset(self.object_list) - STREAM_DATA = getattr(settings, 'STREAM_DATA', False) + STREAM_DATA = getattr(settings, "STREAM_DATA", False) if STREAM_DATA: response = self._get_streaming_response() else: @@ -685,24 +704,25 @@ def _get_streaming_response(self): """ Get a StreamingHttpResponse response object """ + def get_json_string(item): - return json.dumps( - item.json if isinstance(item, Instance) else item) + return json.dumps(item.json if isinstance(item, Instance) else item) - if self.kwargs.get('format') == 'xml': + if self.kwargs.get("format") == "xml": response = StreamingHttpResponse( renderers.InstanceXMLRenderer().stream_data( - self.object_list, self.get_serializer), - content_type="application/xml" + self.object_list, self.get_serializer + ), + content_type="application/xml", ) else: response = StreamingHttpResponse( json_stream(self.object_list, get_json_string), - content_type="application/json" + content_type="application/json", ) # calculate etag value and add it to response headers - if hasattr(self, 'etag_hash'): + if hasattr(self, "etag_hash"): self.set_etag_header(None, self.etag_hash) # set headers on streaming response diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 5d3c3e5121..8d838b7c09 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -3,6 +3,8 @@ import random from datetime import datetime +import six + from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError @@ -17,7 +19,7 @@ HttpResponseRedirect, StreamingHttpResponse, ) -from django.utils import six, timezone +from django.utils import timezone from django.utils.http import urlencode from django.utils.translation import ugettext as _ from django.views.decorators.cache import never_cache diff --git a/onadata/apps/logger/tests/test_transfer_project_command.py b/onadata/apps/logger/tests/test_transfer_project_command.py index 0948627d7b..4339987248 100644 --- a/onadata/apps/logger/tests/test_transfer_project_command.py +++ b/onadata/apps/logger/tests/test_transfer_project_command.py @@ -4,7 +4,7 @@ from django.contrib.auth import get_user_model from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO from onadata.apps.logger.models import Project, XForm from onadata.apps.main.tests.test_base import TestBase @@ -12,87 +12,94 @@ class TestMoveProjectToAnewOwner(TestBase): # pylint: disable=C0111 def test_successful_project_transfer(self): # pylint: disable=C0103 - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, + stdout=mock_stdout, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 0, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 5, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(0, Project.objects.filter(organization=user1).count()) + self.assertEqual(5, Project.objects.filter(organization=user2).count()) def test_single_project_transfer(self): - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) test_project = Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout self.assertIsNotNone(test_project.id) call_command( - 'transferproject', current_owner='user1', new_owner='user2', - project_id=test_project.id + "transferproject", + current_owner="user1", + new_owner="user2", + project_id=test_project.id, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 4, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 1, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(4, Project.objects.filter(organization=user1).count()) + self.assertEqual(1, Project.objects.filter(organization=user2).count()) test_project_refetched = Project.objects.get(id=test_project.id) self.assertEqual(user2, test_project_refetched.organization) @@ -100,12 +107,12 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 """Test the transfer of ownership of the XForms.""" xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xls" + "../fixtures/tutorial/tutorial.xls", ) self._publish_xls_file_and_set_xform(xls_file_path) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) @@ -113,22 +120,22 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 user_model = get_user_model() user_data = { - 'username': 'user', - 'email': 'user@test.com', - 'password': 'test_pass' + "username": "user", + "email": "user@test.com", + "password": "test_pass", } new_owner = user_model.objects.create_user(**user_data) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='bob', new_owner='user', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="bob", + new_owner="user", + all_projects=True, + stdout=mock_stdout, ) - self.assertIn( - 'Projects transferred successfully\n', - mock_stdout.getvalue() - ) - bob = user_model.objects.get(username='bob') + self.assertIn("Projects transferred successfully\n", mock_stdout.getvalue()) + bob = user_model.objects.get(username="bob") bobs_forms = XForm.objects.filter(user=bob) new_owner_forms = XForm.objects.filter(user=new_owner) self.assertEqual(0, bobs_forms.count()) @@ -142,14 +149,16 @@ class TestUserValidation(TestBase): it's stdout is interfering with the other functions causing them to fail. stdout.flush() does not help. """ - def test_user_given_does_not_exist(self): # pylint: disable=C0103 + + def test_user_given_does_not_exist(self): # pylint: disable=C0103 """Test that users are validated before initiating project transfer""" mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, ) - expected_output = 'User user1 does not existUser user2 does '\ - 'not exist\n' + expected_output = "User user1 does not existUser user2 does " "not exist\n" self.assertEqual(mock_stdout.getvalue(), expected_output) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index cef962db4c..11b3a9e849 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -9,6 +9,7 @@ from datetime import datetime import pytz +import six from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -16,16 +17,18 @@ from django.contrib.sites.models import Site from django.core.files import File from django.core.files.storage import get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseRedirect, +) from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader from django.urls import reverse -from django.utils import six from django.utils.translation import ugettext as _ from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import (require_GET, require_http_methods, - require_POST) +from django.views.decorators.http import require_GET, require_http_methods, require_POST from django_digest import HttpDigestAuthenticator from onadata.apps.logger.import_tools import import_instances_from_zip @@ -36,21 +39,28 @@ from onadata.libs.exceptions import EnketoError from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.cache_tools import (cache, - USER_PROFILE_PREFIX) +from onadata.libs.utils.cache_tools import cache, USER_PROFILE_PREFIX from onadata.libs.utils.logger_tools import ( - BaseOpenRosaResponse, OpenRosaResponse, OpenRosaResponseBadRequest, - PublishXForm, inject_instanceid, publish_form, remove_xform, - response_with_mimetype_and_name, safe_create_instance) + BaseOpenRosaResponse, + OpenRosaResponse, + OpenRosaResponseBadRequest, + PublishXForm, + inject_instanceid, + publish_form, + remove_xform, + response_with_mimetype_and_name, + safe_create_instance, +) from onadata.libs.utils.user_auth import ( - HttpResponseNotAuthorized, add_cors_headers, has_edit_permission, - has_permission, helper_auth_helper) -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form, get_form_url) + HttpResponseNotAuthorized, + add_cors_headers, + has_edit_permission, + has_permission, + helper_auth_helper, +) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form, get_form_url -IO_ERROR_STRINGS = [ - 'request data read error', 'error during read(65536) on wsgi.input' -] +IO_ERROR_STRINGS = ["request data read error", "error during read(65536) on wsgi.input"] def _bad_request(e): @@ -60,7 +70,7 @@ def _bad_request(e): def _extract_uuid(text): - text = text[text.find("@key="):-1].replace("@key=", "") + text = text[text.find("@key=") : -1].replace("@key=", "") if text.startswith("uuid:"): text = text.replace("uuid:", "") return text @@ -75,24 +85,23 @@ def _parse_int(num): def _html_submission_response(request, instance): data = {} - data['username'] = instance.xform.user.username - data['id_string'] = instance.xform.id_string - data['domain'] = Site.objects.get(id=settings.SITE_ID).domain + data["username"] = instance.xform.user.username + data["id_string"] = instance.xform.id_string + data["domain"] = Site.objects.get(id=settings.SITE_ID).domain return render(request, "submission.html", data) def _submission_response(instance): data = {} - data['message'] = _("Successful submission.") - data['formid'] = instance.xform.id_string - data['encrypted'] = instance.xform.encrypted - data['instanceID'] = u'uuid:%s' % instance.uuid - data['submissionDate'] = instance.date_created.isoformat() - data['markedAsCompleteDate'] = instance.date_modified.isoformat() + data["message"] = _("Successful submission.") + data["formid"] = instance.xform.id_string + data["encrypted"] = instance.xform.encrypted + data["instanceID"] = "uuid:%s" % instance.uuid + data["submissionDate"] = instance.date_created.isoformat() + data["markedAsCompleteDate"] = instance.date_modified.isoformat() - return BaseOpenRosaResponse( - loader.get_template('submission.xml').render(data)) + return BaseOpenRosaResponse(loader.get_template("submission.xml").render(data)) @require_POST @@ -112,25 +121,30 @@ def bulksubmission(request, username): temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest( - _(u"There was a problem receiving your " - u"ODK submission. [Error: IO Error " - u"reading data]")) + _( + "There was a problem receiving your " + "ODK submission. [Error: IO Error " + "reading data]" + ) + ) if len(temp_postfile) != 1: return HttpResponseBadRequest( - _(u"There was a problem receiving your" - u" ODK submission. [Error: multiple " - u"submission files (?)]")) + _( + "There was a problem receiving your" + " ODK submission. [Error: multiple " + "submission files (?)]" + ) + ) postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) - with open(our_tfpath, 'wb') as f: + with open(our_tfpath, "wb") as f: f.write(postfile.read()) - with open(our_tfpath, 'rb') as f: - total_count, success_count, errors = import_instances_from_zip( - f, posting_user) + with open(our_tfpath, "rb") as f: + total_count, success_count, errors = import_instances_from_zip(f, posting_user) # chose the try approach as suggested by the link below # http://stackoverflow.com/questions/82831 try: @@ -138,22 +152,31 @@ def bulksubmission(request, username): except IOError: pass json_msg = { - 'message': _(u"Submission complete. Out of %(total)d " - u"survey instances, %(success)d were imported, " - u"(%(rejected)d were rejected as duplicates, " - u"missing forms, etc.)") % { - 'total': total_count, - 'success': success_count, - 'rejected': total_count - success_count - }, - 'errors': u"%d %s" % (len(errors), errors) + "message": _( + "Submission complete. Out of %(total)d " + "survey instances, %(success)d were imported, " + "(%(rejected)d were rejected as duplicates, " + "missing forms, etc.)" + ) + % { + "total": total_count, + "success": success_count, + "rejected": total_count - success_count, + }, + "errors": "%d %s" % (len(errors), errors), } audit = {"bulk_submission_log": json_msg} - audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user, - _("Made bulk submissions."), audit, request) + audit_log( + Actions.USER_BULK_SUBMISSION, + request.user, + posting_user, + _("Made bulk submissions."), + audit, + request, + ) response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response @@ -164,9 +187,9 @@ def bulksubmission_form(request, username=None): """ username = username if username is None else username if request.user.username == username: - return render(request, 'bulk_submission_form.html') + return render(request, "bulk_submission_form.html") - return HttpResponseRedirect('/%s' % request.user.username) + return HttpResponseRedirect("/%s" % request.user.username) @require_GET @@ -175,11 +198,11 @@ def formList(request, username): # pylint: disable=C0103 formList view, /formList OpenRosa Form Discovery API 1.0. """ formlist_user = get_object_or_404(User, username__iexact=username) - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -195,33 +218,37 @@ def formList(request, username): # pylint: disable=C0103 # for users who are non-owner if request.user.username == profile.user.username: xforms = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True, - user__username__iexact=username) + downloadable=True, deleted_at__isnull=True, user__username__iexact=username + ) else: xforms = XForm.objects.filter( downloadable=True, deleted_at__isnull=True, user__username__iexact=username, - require_auth=False) + require_auth=False, + ) audit = {} - audit_log(Actions.USER_FORMLIST_REQUESTED, request.user, formlist_user, - _("Requested forms list."), audit, request) + audit_log( + Actions.USER_FORMLIST_REQUESTED, + request.user, + formlist_user, + _("Requested forms list."), + audit, + request, + ) data = { - 'host': request.build_absolute_uri().replace(request.get_full_path(), - ''), - 'xforms': xforms + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "xforms": xforms, } response = render( - request, - "xformsList.xml", - data, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + request, "xformsList.xml", data, content_type="text/xml; charset=utf-8" + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response @@ -231,18 +258,15 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 """ XFormManifest view, part of OpenRosa Form Discovery API 1.0. """ - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) formlist_user = xform.user - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -251,16 +275,17 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 response = render( request, - "xformsManifest.xml", { - 'host': - request.build_absolute_uri().replace(request.get_full_path(), ''), - 'media_files': - MetaData.media_upload(xform, download=True) + "xformsManifest.xml", + { + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "media_files": MetaData.media_upload(xform, download=True), }, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + content_type="text/xml; charset=utf-8", + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response @@ -280,14 +305,16 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 if not authenticator.authenticate(request): return authenticator.build_challenge_response() - if request.method == 'HEAD': + if request.method == "HEAD": response = OpenRosaResponse(status=204) if username: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/submission' % username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/%s/submission" % username + ) else: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/submission') + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/submission" + ) return response xml_file_list = [] @@ -299,27 +326,33 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 xml_file_list = request.FILES.pop("xml_submission_file", []) if len(xml_file_list) != 1: return OpenRosaResponseBadRequest( - _(u"There should be a single XML submission file.")) + _("There should be a single XML submission file.") + ) # save this XML file and media files as attachments media_files = request.FILES.values() # get uuid from post request - uuid = request.POST.get('uuid') + uuid = request.POST.get("uuid") - error, instance = safe_create_instance(username, xml_file_list[0], - media_files, uuid, request) + error, instance = safe_create_instance( + username, xml_file_list[0], media_files, uuid, request + ) if error: return error elif instance is None: - return OpenRosaResponseBadRequest( - _(u"Unable to create submission.")) + return OpenRosaResponseBadRequest(_("Unable to create submission.")) audit = {"xform": instance.xform.id_string} - audit_log(Actions.SUBMISSION_CREATED, request.user, - instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, request) + audit_log( + Actions.SUBMISSION_CREATED, + request.user, + instance.xform.user, + _("Created submission on form %(id_string)s.") + % {"id_string": instance.xform.id_string}, + audit, + request, + ) # response as html if posting with a UUID if not username and uuid: @@ -331,12 +364,11 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 # 1) the status code needs to be 201 (created) # 2) The location header needs to be set to the host it posted to response.status_code = 201 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response except IOError as e: if _bad_request(e): - return OpenRosaResponseBadRequest( - _(u"File transfer interruption.")) + return OpenRosaResponseBadRequest(_("File transfer interruption.")) else: raise finally: @@ -351,7 +383,7 @@ def download_xform(request, username, id_string): Download XForm XML view. """ user = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': user, 'id_string__iexact': id_string}) + xform = get_form({"user": user, "id_string__iexact": id_string}) profile, __ = UserProfile.objects.get_or_create(user=user) if profile.require_auth: @@ -359,11 +391,15 @@ def download_xform(request, username, id_string): if not authenticator.authenticate(request): return authenticator.build_challenge_response() audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XML_DOWNLOADED, request.user, xform.user, - _("Downloaded XML for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) - response = response_with_mimetype_and_name( - 'xml', id_string, show_date=False) + audit_log( + Actions.FORM_XML_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XML for form '%(id_string)s'.") % {"id_string": xform.id_string}, + audit, + request, + ) + response = response_with_mimetype_and_name("xml", id_string, show_date=False) response.content = xform.xml return response @@ -372,44 +408,53 @@ def download_xlsform(request, username, id_string): """ Download XLSForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = User.objects.get(username__iexact=username) helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") file_path = xform.xls.name default_storage = get_storage_class()() - if file_path != '' and default_storage.exists(file_path): + if file_path != "" and default_storage.exists(file_path): audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XLS_DOWNLOADED, request.user, xform.user, - _("Downloaded XLS file for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) + audit_log( + Actions.FORM_XLS_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XLS file for form '%(id_string)s'.") + % {"id_string": xform.id_string}, + audit, + request, + ) split_path = file_path.split(os.extsep) - extension = 'xls' + extension = "xls" if len(split_path) > 1: extension = split_path[len(split_path) - 1] response = response_with_mimetype_and_name( - 'vnd.ms-excel', + "vnd.ms-excel", id_string, show_date=False, extension=extension, - file_path=file_path) + file_path=file_path, + ) return response else: - messages.add_message(request, messages.WARNING, - _(u'No XLS file for your form ' - u'%(id)s') % {'id': id_string}) + messages.add_message( + request, + messages.WARNING, + _("No XLS file for your form " "%(id)s") + % {"id": id_string}, + ) return HttpResponseRedirect("/%s" % username) @@ -419,10 +464,9 @@ def download_jsonform(request, username, id_string): XForm JSON view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if request.method == "OPTIONS": response = HttpResponse() @@ -430,13 +474,12 @@ def download_jsonform(request, username, id_string): return response helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - response = HttpResponseForbidden(_(u'Not shared.')) + response = HttpResponseForbidden(_("Not shared.")) add_cors_headers(response) return response - response = response_with_mimetype_and_name( - 'json', id_string, show_date=False) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') + response = response_with_mimetype_and_name("json", id_string, show_date=False) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") response.content = "%s(%s)" % (callback, xform.json) else: add_cors_headers(response) @@ -450,20 +493,26 @@ def delete_xform(request, username, id_string): """ Delete XForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) # delete xform and submissions remove_xform(xform) audit = {} - audit_log(Actions.FORM_DELETED, request.user, xform.user, - _("Deleted form '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect('/') + audit_log( + Actions.FORM_DELETED, + request.user, + xform.user, + _("Deleted form '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + return HttpResponseRedirect("/") @is_owner @@ -471,21 +520,26 @@ def toggle_downloadable(request, username, id_string): """ Toggle XForm view, changes downloadable status of a form. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) xform.downloadable = not xform.downloadable xform.save() audit = {} - audit_log(Actions.FORM_UPDATED, request.user, xform.user, - _("Made form '%(id_string)s' %(downloadable)s.") % { - 'id_string': - xform.id_string, - 'downloadable': - _("downloadable") - if xform.downloadable else _("un-downloadable") - }, audit, request) + audit_log( + Actions.FORM_UPDATED, + request.user, + xform.user, + _("Made form '%(id_string)s' %(downloadable)s.") + % { + "id_string": xform.id_string, + "downloadable": _("downloadable") + if xform.downloadable + else _("un-downloadable"), + }, + audit, + request, + ) return HttpResponseRedirect("/%s" % username) @@ -494,48 +548,47 @@ def enter_data(request, username, id_string): Redirects to Enketo webform view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: enketo_urls = get_enketo_urls(form_url, xform.id_string) - url = enketo_urls.get('url') + url = enketo_urls.get("url") if not url: return HttpResponseRedirect( reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + "form-show", kwargs={"username": username, "id_string": id_string} + ) + ) return HttpResponseRedirect(url) except EnketoError as e: data = {} owner = User.objects.get(username__iexact=username) - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - data['xform'] = xform - data['content_user'] = owner - data['form_view'] = True - data['message'] = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + data["xform"] = xform + data["content_user"] = owner + data["form_view"] = True + data["message"] = { + "type": "alert-error", + "text": "Enketo error, reason: %s" % e, } messages.add_message( request, messages.WARNING, _("Enketo error: enketo replied %s") % e, - fail_silently=True) + fail_silently=True, + ) return render(request, "profile.html", data) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def edit_data(request, username, id_string, data_id): @@ -544,30 +597,27 @@ def edit_data(request, username, id_string, data_id): """ context = RequestContext(request) owner = User.objects.get(username__iexact=username) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) instance = get_object_or_404(Instance, pk=data_id, xform=xform) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) - if not hasattr(settings, 'ENKETO_URL'): + return HttpResponseForbidden(_("Not shared.")) + if not hasattr(settings, "ENKETO_URL"): return HttpResponseRedirect( - reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) - url = '%sdata/edit_url' % settings.ENKETO_URL + url = "%sdata/edit_url" % settings.ENKETO_URL # see commit 220f2dad0e for tmp file creation injected_xml = inject_instanceid(instance.xml, instance.uuid) return_url = request.build_absolute_uri( reverse( - 'submission-instance', - kwargs={'username': username, - 'id_string': id_string}) + "#/" + text(instance.id)) + "submission-instance", kwargs={"username": username, "id_string": id_string} + ) + + "#/" + + text(instance.id) + ) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: @@ -576,26 +626,27 @@ def edit_data(request, username, id_string, data_id): xform.id_string, instance_xml=injected_xml, instance_id=instance.uuid, - return_url=return_url) + return_url=return_url, + ) except EnketoError as e: context.message = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + "type": "alert-error", + "text": "Enketo error, reason: %s" % e, } messages.add_message( request, messages.WARNING, _("Enketo error: enketo replied %s") % e, - fail_silently=True) + fail_silently=True, + ) else: if url: - url = url['edit_url'] + url = url["edit_url"] context.enketo = url return HttpResponseRedirect(url) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def view_submission_list(request, username): @@ -609,18 +660,15 @@ def view_submission_list(request, username): authenticator = HttpDigestAuthenticator() if not authenticator.authenticate(request): return authenticator.build_challenge_response() - id_string = request.GET.get('formId', None) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + id_string = request.GET.get("formId", None) + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') - num_entries = request.GET.get('numEntries', None) - cursor = request.GET.get('cursor', None) - instances = xform.instances.filter(deleted_at=None).order_by('pk') + return HttpResponseForbidden("Not shared.") + num_entries = request.GET.get("numEntries", None) + cursor = request.GET.get("cursor", None) + instances = xform.instances.filter(deleted_at=None).order_by("pk") cursor = _parse_int(cursor) if cursor: @@ -630,7 +678,7 @@ def view_submission_list(request, username): if num_entries: instances = instances[:num_entries] - data = {'instances': instances} + data = {"instances": instances} resumption_cursor = 0 if instances.count(): @@ -639,13 +687,11 @@ def view_submission_list(request, username): elif instances.count() == 0 and cursor: resumption_cursor = cursor - data['resumptionCursor'] = resumption_cursor + data["resumptionCursor"] = resumption_cursor return render( - request, - 'submissionList.xml', - data, - content_type="text/xml; charset=utf-8") + request, "submissionList.xml", data, content_type="text/xml; charset=utf-8" + ) def view_download_submission(request, username): @@ -660,12 +706,12 @@ def view_download_submission(request, username): if not authenticator.authenticate(request): return authenticator.build_challenge_response() data = {} - form_id = request.GET.get('formId', None) + form_id = request.GET.get("formId", None) if not isinstance(form_id, six.string_types): return HttpResponseBadRequest() - id_string = form_id[0:form_id.find('[')] - form_id_parts = form_id.split('/') + id_string = form_id[0 : form_id.find("[")] + form_id_parts = form_id.split("/") if form_id_parts.__len__() < 2: return HttpResponseBadRequest() @@ -675,25 +721,23 @@ def view_download_submission(request, username): xform__id_string__iexact=id_string, uuid=uuid, xform__user__username=username, - deleted_at__isnull=True) + deleted_at__isnull=True, + ) xform = instance.xform if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") submission_xml_root_node = instance.get_root_node() - submission_xml_root_node.setAttribute('instanceID', - u'uuid:%s' % instance.uuid) - submission_xml_root_node.setAttribute('submissionDate', - instance.date_created.isoformat()) - data['submission_data'] = submission_xml_root_node.toxml() - data['media_files'] = Attachment.objects.filter(instance=instance) - data['host'] = request.build_absolute_uri().replace( - request.get_full_path(), '') + submission_xml_root_node.setAttribute("instanceID", "uuid:%s" % instance.uuid) + submission_xml_root_node.setAttribute( + "submissionDate", instance.date_created.isoformat() + ) + data["submission_data"] = submission_xml_root_node.toxml() + data["media_files"] = Attachment.objects.filter(instance=instance) + data["host"] = request.build_absolute_uri().replace(request.get_full_path(), "") return render( - request, - 'downloadSubmission.xml', - data, - content_type="text/xml; charset=utf-8") + request, "downloadSubmission.xml", data, content_type="text/xml; charset=utf-8" + ) @require_http_methods(["HEAD", "POST"]) @@ -711,23 +755,27 @@ def form_upload(request, username): return authenticator.build_challenge_response() if form_user != request.user: return HttpResponseForbidden( - _(u"Not allowed to upload form[s] to %(user)s account." % - {'user': form_user})) - if request.method == 'HEAD': + _( + "Not allowed to upload form[s] to %(user)s account." + % {"user": form_user} + ) + ) + if request.method == "HEAD": response = OpenRosaResponse(status=204) - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/formUpload' % form_user.username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/%s/formUpload" % form_user.username + ) return response - xform_def = request.FILES.get('form_def_file', None) - content = u"" + xform_def = request.FILES.get("form_def_file", None) + content = "" if isinstance(xform_def, File): do_form_upload = PublishXForm(xform_def, form_user) xform = publish_form(do_form_upload.publish_xform) status = 201 if isinstance(xform, XForm): - content = _(u"%s successfully published." % xform.id_string) + content = _("%s successfully published." % xform.id_string) else: - content = xform['text'] + content = xform["text"] if isinstance(content, Exception): content = content status = 500 diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 9c14884361..8be56809c6 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,3 +1,5 @@ +import six + from uuid import UUID from django.contrib.auth.models import User @@ -6,7 +8,6 @@ from django.db.models import Q from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.utils import six from django_filters import rest_framework as django_filter_filters from rest_framework import filters from rest_framework_guardian.filters import ObjectPermissionsFilter @@ -16,18 +17,15 @@ from onadata.apps.viewer.models import Export from onadata.libs.utils.numeric import int_or_parse_error from onadata.libs.utils.common_tags import MEDIA_FILE_TYPES -from onadata.libs.permissions import \ - exclude_items_from_queryset_using_xform_meta_perms +from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms class AnonDjangoObjectPermissionFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): """ Anonymous user has no object permissions, return queryset as it is. """ - form_id = view.kwargs.get( - view.lookup_field, view.kwargs.get('xform_pk')) + form_id = view.kwargs.get(view.lookup_field, view.kwargs.get("xform_pk")) lookup_field = view.lookup_field queryset = queryset.filter(deleted_at=None) @@ -35,16 +33,15 @@ def filter_queryset(self, request, queryset, view): return queryset if form_id: - if lookup_field == 'pk': - int_or_parse_error(form_id, - u'Invalid form ID. It must be a positive' - ' integer') + if lookup_field == "pk": + int_or_parse_error( + form_id, "Invalid form ID. It must be a positive" " integer" + ) try: - if lookup_field == 'uuid': + if lookup_field == "uuid": form_id = UUID(form_id) - form = queryset.get( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + form = queryset.get(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) else: xform_kwargs = {lookup_field: form_id} form = queryset.get(**xform_kwargs) @@ -53,14 +50,14 @@ def filter_queryset(self, request, queryset, view): # Check if form is public and return it if form.shared: - if lookup_field == 'uuid': - return queryset.filter( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + if lookup_field == "uuid": + return queryset.filter(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) else: return queryset.filter(Q(**xform_kwargs)) - return super(AnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + return super(AnonDjangoObjectPermissionFilter, self).filter_queryset( + request, queryset, view + ) # pylint: disable=too-few-public-methods @@ -73,20 +70,22 @@ class EnketoAnonDjangoObjectPermissionFilter(AnonDjangoObjectPermissionFilter): def filter_queryset(self, request, queryset, view): """Check report_xform permission when requesting for Enketo URL.""" - if view.action == 'enketo': - self.perm_format = '%(app_label)s.report_%(model_name)s' # noqa pylint: disable=W0201 - return super(EnketoAnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + if view.action == "enketo": + self.perm_format = ( + "%(app_label)s.report_%(model_name)s" # noqa pylint: disable=W0201 + ) + return super(EnketoAnonDjangoObjectPermissionFilter, self).filter_queryset( + request, queryset, view + ) class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): - perm_format = '%(app_label)s.report_%(model_name)s' + perm_format = "%(app_label)s.report_%(model_name)s" class XFormListXFormPKFilter(object): - def filter_queryset(self, request, queryset, view): - xform_pk = view.kwargs.get('xform_pk') + xform_pk = view.kwargs.get("xform_pk") if xform_pk: try: xform_pk = int(xform_pk) @@ -105,38 +104,36 @@ class FormIDFilter(django_filter_filters.FilterSet): class Meta: model = XForm - fields = ['formID'] + fields = ["formID"] class OrganizationPermissionFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): """Return a filtered queryset or all profiles if a getting a specific - profile.""" - if view.action == 'retrieve' and request.method == 'GET': + profile.""" + if view.action == "retrieve" and request.method == "GET": return queryset.model.objects.all() - filtered_queryset = super(OrganizationPermissionFilter, self)\ - .filter_queryset(request, queryset, view) - org_users = set([group.team.organization - for group in request.user.groups.all()] + [ - o.user for o in filtered_queryset]) + filtered_queryset = super(OrganizationPermissionFilter, self).filter_queryset( + request, queryset, view + ) + org_users = set( + [group.team.organization for group in request.user.groups.all()] + + [o.user for o in filtered_queryset] + ) - return queryset.model.objects.filter(user__in=org_users, - user__is_active=True) + return queryset.model.objects.filter(user__in=org_users, user__is_active=True) class XFormOwnerFilter(filters.BaseFilterBackend): - owner_prefix = 'user' + owner_prefix = "user" def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) @@ -144,7 +141,6 @@ def filter_queryset(self, request, queryset, view): class DataFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): if request.user.is_anonymous: return queryset.filter(Q(shared_data=True)) @@ -155,57 +151,76 @@ class InstanceFilter(django_filter_filters.FilterSet): """ Instance FilterSet implemented using django-filter """ + submitted_by__id = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='id', + to_field_name="id", ) submitted_by__username = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='username', + to_field_name="username", ) media_all_received = django_filter_filters.BooleanFilter() class Meta: model = Instance - date_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte', 'year', - 'year__gt', 'year__lt', 'year__gte', 'year__lte', - 'month', 'month__gt', 'month__lt', 'month__gte', - 'month__lte', 'day', 'day__gt', 'day__lt', - 'day__gte', 'day__lte'] - generic_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte'] - fields = {'date_created': date_field_lookups, - 'date_modified': date_field_lookups, - 'last_edited': date_field_lookups, - 'media_all_received': ['exact'], - 'status': ['exact'], - 'survey_type__slug': ['exact'], - 'user__id': ['exact'], - 'user__username': ['exact'], - 'uuid': ['exact'], - 'version': generic_field_lookups} + date_field_lookups = [ + "exact", + "gt", + "lt", + "gte", + "lte", + "year", + "year__gt", + "year__lt", + "year__gte", + "year__lte", + "month", + "month__gt", + "month__lt", + "month__gte", + "month__lte", + "day", + "day__gt", + "day__lt", + "day__gte", + "day__lte", + ] + generic_field_lookups = ["exact", "gt", "lt", "gte", "lte"] + fields = { + "date_created": date_field_lookups, + "date_modified": date_field_lookups, + "last_edited": date_field_lookups, + "media_all_received": ["exact"], + "status": ["exact"], + "survey_type__slug": ["exact"], + "user__id": ["exact"], + "user__username": ["exact"], + "uuid": ["exact"], + "version": generic_field_lookups, + } class ProjectOwnerFilter(filters.BaseFilterBackend): - owner_prefix = 'organization' + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) | Project.objects.filter( - shared=True, deleted_at__isnull=True, **kwargs) + shared=True, deleted_at__isnull=True, **kwargs + ) return queryset class AnonUserProjectFilter(ObjectPermissionsFilter): - owner_prefix = 'organization' + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): """ @@ -218,9 +233,10 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(Q(shared=True)) if project_id: - int_or_parse_error(project_id, - u"Invalid value for project_id. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for project_id. It must be a" " positive integer.", + ) # check if project is public and return it try: @@ -231,38 +247,36 @@ def filter_queryset(self, request, queryset, view): if project.shared: return queryset.filter(Q(id=project_id)) - return super(AnonUserProjectFilter, self)\ - .filter_queryset(request, queryset, view) + return super(AnonUserProjectFilter, self).filter_queryset( + request, queryset, view + ) class TagFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): # filter by tags if available. - tags = request.query_params.get('tags', None) + tags = request.query_params.get("tags", None) if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") return queryset.filter(tags__name__in=tags) return queryset class XFormPermissionFilterMixin(object): - def _xform_filter(self, request, view, keyword): """Use XForm permissions""" - xform = request.query_params.get('xform') + xform = request.query_params.get("xform") public_forms = XForm.objects.none() if xform: - int_or_parse_error(xform, - u"Invalid value for formid. It must be a" - " positive integer.") + int_or_parse_error( + xform, "Invalid value for formid. It must be a" " positive integer." + ) self.xform = get_object_or_404(XForm, pk=xform) xform_qs = XForm.objects.filter(pk=self.xform.pk) - public_forms = XForm.objects.filter(pk=self.xform.pk, - shared_data=True) + public_forms = XForm.objects.filter(pk=self.xform.pk, shared_data=True) else: xform_qs = XForm.objects.all() xform_qs = xform_qs.filter(deleted_at=None) @@ -270,8 +284,12 @@ def _xform_filter(self, request, view, keyword): if request.user.is_anonymous: xforms = xform_qs.filter(shared_data=True) else: - xforms = super(XFormPermissionFilterMixin, self).filter_queryset( - request, xform_qs, view) | public_forms + xforms = ( + super(XFormPermissionFilterMixin, self).filter_queryset( + request, xform_qs, view + ) + | public_forms + ) return {"%s__in" % keyword: xforms} def _xform_filter_queryset(self, request, queryset, view, keyword): @@ -280,14 +298,14 @@ def _xform_filter_queryset(self, request, queryset, view, keyword): class ProjectPermissionFilterMixin(object): - def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") if project_id: - int_or_parse_error(project_id, - u"Invalid value for projectid. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for projectid. It must be a" " positive integer.", + ) project = get_object_or_404(Project, pk=project_id) project_qs = Project.objects.filter(pk=project.id) @@ -295,7 +313,8 @@ def _project_filter(self, request, view, keyword): project_qs = Project.objects.all() projects = super(ProjectPermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + request, project_qs, view + ) return {"%s__in" % keyword: projects} @@ -307,7 +326,6 @@ def _project_filter_queryset(self, request, queryset, view, keyword): class InstancePermissionFilterMixin(object): - def _instance_filter(self, request, view, keyword): instance_kwarg = {} instance_content_type = ContentType.objects.get_for_model(Instance) @@ -315,13 +333,14 @@ def _instance_filter(self, request, view, keyword): instance_id = request.query_params.get("instance") project_id = request.query_params.get("project") - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") if instance_id and project_id and xform_id: for object_id in [instance_id, project_id]: - int_or_parse_error(object_id, - u"Invalid value for instanceid. It must be" - " a positive integer.") + int_or_parse_error( + object_id, + "Invalid value for instanceid. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) # test if user has permissions on the project @@ -337,9 +356,9 @@ def _instance_filter(self, request, view, keyword): project_qs = Project.objects.filter(pk=project.id) if parent and parent.project == project: - projects = super( - InstancePermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + projects = super(InstancePermissionFilterMixin, self).filter_queryset( + request, project_qs, view + ) instances = [instance.id] if projects else [] @@ -359,23 +378,21 @@ def _instance_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) -class RestServiceFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class RestServiceFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - return self._xform_filter_queryset( - request, queryset, view, 'xform_id') + return self._xform_filter_queryset(request, queryset, view, "xform_id") -class MetaDataFilter(ProjectPermissionFilterMixin, - InstancePermissionFilterMixin, - XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class MetaDataFilter( + ProjectPermissionFilterMixin, + InstancePermissionFilterMixin, + XFormPermissionFilterMixin, + ObjectPermissionsFilter, +): def filter_queryset(self, request, queryset, view): keyword = "object_id" - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") project_id = request.query_params.get("project") instance_id = request.query_params.get("instance") @@ -392,8 +409,11 @@ def filter_queryset(self, request, queryset, view): # return instance specific metadata if instance_id: - return (queryset.filter(Q(**instance_kwarg)) - if (xform_id and instance_kwarg) else []) + return ( + queryset.filter(Q(**instance_kwarg)) + if (xform_id and instance_kwarg) + else [] + ) elif xform_id: # return xform specific metadata return queryset.filter(Q(**xform_kwarg)) @@ -403,31 +423,32 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(Q(**project_kwarg)) # return all project,instance and xform metadata information - return queryset.filter(Q(**xform_kwarg) | Q(**project_kwarg) | - Q(**instance_kwarg)) - + return queryset.filter( + Q(**xform_kwarg) | Q(**project_kwarg) | Q(**instance_kwarg) + ) -class AttachmentFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +class AttachmentFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - queryset = self._xform_filter_queryset(request, queryset, view, - 'instance__xform') + queryset = self._xform_filter_queryset( + request, queryset, view, "instance__xform" + ) # Ensure queryset is filtered by XForm meta permissions - xform_ids = set( - queryset.values_list("instance__xform", flat=True)) + xform_ids = set(queryset.values_list("instance__xform", flat=True)) for xform_id in xform_ids: xform = XForm.objects.get(id=xform_id) user = request.user queryset = exclude_items_from_queryset_using_xform_meta_perms( - xform, user, queryset) + xform, user, queryset + ) - instance_id = request.query_params.get('instance') + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer.") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -435,9 +456,8 @@ def filter_queryset(self, request, queryset, view): class AttachmentTypeFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - attachment_type = request.query_params.get('type') + attachment_type = request.query_params.get("type") mime_types = MEDIA_FILE_TYPES.get(attachment_type) @@ -448,15 +468,12 @@ def filter_queryset(self, request, queryset, view): class TeamOrgFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - org = request.data.get('org') or request.query_params.get('org') + org = request.data.get("org") or request.query_params.get("org") # Get all the teams for the organization if org: - kwargs = { - 'organization__username__iexact': org - } + kwargs = {"organization__username__iexact": org} return Team.objects.filter(**kwargs) @@ -464,26 +481,24 @@ def filter_queryset(self, request, queryset, view): class UserNoOrganizationsFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - if str(request.query_params.get('orgs')).lower() == 'false': + if str(request.query_params.get("orgs")).lower() == "false": organization_user_ids = OrganizationProfile.objects.values_list( - 'user__id', - flat=True) + "user__id", flat=True + ) queryset = queryset.exclude(id__in=organization_user_ids) return queryset class OrganizationsSharedWithUserFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): """ This returns a queryset containing only organizations to which the passed user belongs. """ - username = request.query_params.get('shared_with') + username = request.query_params.get("shared_with") if username: try: @@ -491,17 +506,14 @@ def filter_queryset(self, request, queryset, view): # Groups a User belongs to are available as a queryset property # of a User object, which this code takes advantage of - organization_user_ids = User.objects\ - .get(username=username)\ - .groups\ - .all()\ - .values_list( - 'team__organization', - flat=True)\ - .distinct() + organization_user_ids = ( + User.objects.get(username=username) + .groups.all() + .values_list("team__organization", flat=True) + .distinct() + ) - filtered_queryset = queryset.filter( - user_id__in=organization_user_ids) + filtered_queryset = queryset.filter(user_id__in=organization_user_ids) return filtered_queryset @@ -511,27 +523,22 @@ def filter_queryset(self, request, queryset, view): return queryset -class WidgetFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class WidgetFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - if view.action == 'list': + if view.action == "list": # Return widgets from xform user has perms to - return self._xform_filter_queryset(request, queryset, view, - 'object_id') + return self._xform_filter_queryset(request, queryset, view, "object_id") - return super(WidgetFilter, self).filter_queryset(request, queryset, - view) + return super(WidgetFilter, self).filter_queryset(request, queryset, view) class UserProfileFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - if view.action == 'list': - users = request.GET.get('users') + if view.action == "list": + users = request.GET.get("users") if users: - users = users.split(',') + users = users.split(",") return queryset.filter(user__username__in=users) elif not request.user.is_anonymous: return queryset.filter(user__username=request.user.username) @@ -543,12 +550,13 @@ def filter_queryset(self, request, queryset, view): class NoteFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): - instance_id = request.query_params.get('instance') + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -556,8 +564,7 @@ def filter_queryset(self, request, queryset, view): return queryset -class ExportFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +class ExportFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): """ ExportFilter class uses permissions on the related xform to filter Export queryesets. Also filters submitted_by a specific user. @@ -572,29 +579,30 @@ def _is_public_xform(self, export_id: int): return False def filter_queryset(self, request, queryset, view): - has_submitted_by_key = (Q(options__has_key='query') & - Q(options__query__has_key='_submitted_by'),) + has_submitted_by_key = ( + Q(options__has_key="query") & Q(options__query__has_key="_submitted_by"), + ) - if request.user.is_anonymous or self._is_public_xform( - view.kwargs.get('pk')): + if request.user.is_anonymous or self._is_public_xform(view.kwargs.get("pk")): return self._xform_filter_queryset( - request, queryset, view, 'xform_id')\ - .exclude(*has_submitted_by_key) + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) old_perm_format = self.perm_format # only if request.user has access to all data - self.perm_format = old_perm_format + '_all' - all_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .exclude(*has_submitted_by_key) + self.perm_format = old_perm_format + "_all" + all_qs = self._xform_filter_queryset( + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) # request.user has access to own submitted data - self.perm_format = old_perm_format + '_data' - submitter_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .filter(*has_submitted_by_key)\ + self.perm_format = old_perm_format + "_data" + submitter_qs = ( + self._xform_filter_queryset(request, queryset, view, "xform_id") + .filter(*has_submitted_by_key) .filter(options__query___submitted_by=request.user.username) + ) return all_qs | submitter_qs diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index fe6711a66b..fd508a74d9 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -9,7 +9,9 @@ from typing import Tuple import pytz -from django.utils import six, timezone +import six + +from django.utils import timezone from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index ef855641da..f52bc3b552 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -6,6 +6,7 @@ import os import sys from datetime import datetime +import six import httplib2 from celery.backends.rpc import BacklogLimitExceeded @@ -13,14 +14,15 @@ from django.conf import settings from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils import six from django.utils.translation import ugettext as _ from kombu.exceptions import OperationalError from oauth2client import client as google_client -from oauth2client.client import (HttpAccessTokenRefreshError, - OAuth2WebServerFlow, TokenRevokeError) -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage +from oauth2client.client import ( + HttpAccessTokenRefreshError, + OAuth2WebServerFlow, + TokenRevokeError, +) +from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage from requests import ConnectionError from rest_framework import exceptions, status from rest_framework.response import Response @@ -30,43 +32,56 @@ from onadata.apps.main.models import TokenStorageModel from onadata.apps.viewer import tasks as viewer_task from onadata.apps.viewer.models.export import Export, ExportConnectionError -from onadata.libs.exceptions import (J2XException, NoRecordsFoundError, - NoRecordsPermission, ServiceUnavailable) +from onadata.libs.exceptions import ( + J2XException, + NoRecordsFoundError, + NoRecordsPermission, + ServiceUnavailable, +) from onadata.libs.permissions import filter_queryset_xform_meta_perms_sql from onadata.libs.utils import log -from onadata.libs.utils.async_status import (FAILED, PENDING, SUCCESSFUL, - async_status, - celery_state_to_status) -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG, OSM, - SUBMISSION_TIME) +from onadata.libs.utils.async_status import ( + FAILED, + PENDING, + SUCCESSFUL, + async_status, + celery_state_to_status, +) +from onadata.libs.utils.common_tags import ( + DATAVIEW_EXPORT, + GROUPNAME_REMOVED_FLAG, + OSM, + SUBMISSION_TIME, +) from onadata.libs.utils.common_tools import report_exception -from onadata.libs.utils.export_tools import (check_pending_export, - generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export, - newest_export_for, - parse_request_export_options, - should_create_new_export) +from onadata.libs.utils.export_tools import ( + check_pending_export, + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, + newest_export_for, + parse_request_export_options, + should_create_new_export, +) from onadata.libs.utils.logger_tools import response_with_mimetype_and_name from onadata.libs.utils.model_tools import get_columns_with_hxl # Supported external exports -EXTERNAL_EXPORT_TYPES = ['xls'] +EXTERNAL_EXPORT_TYPES = ["xls"] EXPORT_EXT = { - 'xls': Export.XLS_EXPORT, - 'xlsx': Export.XLS_EXPORT, - 'csv': Export.CSV_EXPORT, - 'csvzip': Export.CSV_ZIP_EXPORT, - 'savzip': Export.SAV_ZIP_EXPORT, - 'uuid': Export.EXTERNAL_EXPORT, - 'kml': Export.KML_EXPORT, - 'zip': Export.ZIP_EXPORT, + "xls": Export.XLS_EXPORT, + "xlsx": Export.XLS_EXPORT, + "csv": Export.CSV_EXPORT, + "csvzip": Export.CSV_ZIP_EXPORT, + "savzip": Export.SAV_ZIP_EXPORT, + "uuid": Export.EXTERNAL_EXPORT, + "kml": Export.KML_EXPORT, + "zip": Export.ZIP_EXPORT, OSM: Export.OSM_EXPORT, - 'gsheets': Export.GOOGLE_SHEETS_EXPORT + "gsheets": Export.GOOGLE_SHEETS_EXPORT, } @@ -89,53 +104,60 @@ def _get_export_type(export_type): export_type = EXPORT_EXT[export_type] else: raise exceptions.ParseError( - _(u"'%(export_type)s' format not known or not implemented!" % - {'export_type': export_type})) + _( + "'%(export_type)s' format not known or not implemented!" + % {"export_type": export_type} + ) + ) return export_type # pylint: disable=too-many-arguments, too-many-locals, too-many-branches -def custom_response_handler(request, - xform, - query, - export_type, - token=None, - meta=None, - dataview=False, - filename=None): +def custom_response_handler( + request, + xform, + query, + export_type, + token=None, + meta=None, + dataview=False, + filename=None, +): """ Returns a HTTP response with export file for download. """ export_type = _get_export_type(export_type) - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT options = parse_request_export_options(request.query_params) - dataview_pk = hasattr(dataview, 'pk') and dataview.pk + dataview_pk = hasattr(dataview, "pk") and dataview.pk options["dataview_pk"] = dataview_pk if dataview: - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl: - options['include_hxl'] = include_hxl_row(dataview.columns, - list(columns_with_hxl)) + options["include_hxl"] = include_hxl_row( + dataview.columns, list(columns_with_hxl) + ) try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: return Response( - data=json.dumps({ - "details": _("You don't have permission") - }), + data=json.dumps({"details": _("You don't have permission")}), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) if query: - options['query'] = query + options["query"] = query remove_group_name = options.get("remove_group_name") @@ -147,20 +169,21 @@ def custom_response_handler(request, if export_type == Export.GOOGLE_SHEETS_EXPORT: return Response( - data=json.dumps({ - "details": _("Sheets export only supported in async mode") - }), + data=json.dumps( + {"details": _("Sheets export only supported in async mode")} + ), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) # check if we need to re-generate, # we always re-generate if a filter is specified def _new_export(): return _generate_new_export( - request, xform, query, export_type, dataview_pk=dataview_pk) + request, xform, query, export_type, dataview_pk=dataview_pk + ) - if should_create_new_export( - xform, export_type, options, request=request): + if should_create_new_export(xform, export_type, options, request=request): export = _new_export() else: export = newest_export_for(xform, export_type, options) @@ -184,7 +207,8 @@ def _new_export(): show_date = True if filename is None and export.status == Export.SUCCESSFUL: filename = _generate_filename( - request, xform, remove_group_name, dataview_pk=dataview_pk) + request, xform, remove_group_name, dataview_pk=dataview_pk + ) else: show_date = False response = response_with_mimetype_and_name( @@ -192,13 +216,13 @@ def _new_export(): filename, extension=ext, show_date=show_date, - file_path=export.filepath) + file_path=export.filepath, + ) return response -def _generate_new_export(request, xform, query, export_type, - dataview_pk=False): +def _generate_new_export(request, xform, query, export_type, dataview_pk=False): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) @@ -208,18 +232,17 @@ def _generate_new_export(request, xform, query, export_type, "id_string": xform.id_string, } if query: - options['query'] = query + options["query"] = query options["dataview_pk"] = dataview_pk if export_type == Export.GOOGLE_SHEETS_EXPORT: - options['google_credentials'] = \ - _get_google_credential(request).to_json() + options["google_credentials"] = _get_google_credential(request).to_json() try: if export_type == Export.EXTERNAL_EXPORT: - options['token'] = request.GET.get('token') - options['data_id'] = request.GET.get('data_id') - options['meta'] = request.GET.get('meta') + options["token"] = request.GET.get("token") + options["data_id"] = request.GET.get("data_id") + options["meta"] = request.GET.get("meta") export = generate_external_export( export_type, @@ -227,7 +250,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, @@ -235,7 +259,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.ZIP_EXPORT: export = generate_attachments_zip_export( export_type, @@ -243,7 +268,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, @@ -251,7 +277,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) else: options.update(parse_request_export_options(request.query_params)) @@ -259,10 +286,14 @@ def _generate_new_export(request, xform, query, export_type, audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_CREATED, request.user, xform.user, - _("Created %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + log.Actions.EXPORT_CREATED, + request.user, + xform.user, + _("Created %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: @@ -281,10 +312,14 @@ def log_export(request, xform, export_type): # log download as well audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_DOWNLOADED, request.user, xform.user, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + log.Actions.EXPORT_DOWNLOADED, + request.user, + xform.user, + _("Downloaded %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) def external_export_response(export): @@ -292,21 +327,16 @@ def external_export_response(export): Redirects to export_url of XLSReports successful export. In case of a failure, returns a 400 HTTP JSON response with the error message. """ - if isinstance(export, Export) \ - and export.internal_status == Export.SUCCESSFUL: + if isinstance(export, Export) and export.internal_status == Export.SUCCESSFUL: return HttpResponseRedirect(export.export_url) else: http_status = status.HTTP_400_BAD_REQUEST - return Response( - json.dumps(export), http_status, content_type="application/json") + return Response(json.dumps(export), http_status, content_type="application/json") -def _generate_filename(request, - xform, - remove_group_name=False, - dataview_pk=False): - if request.GET.get('raw'): +def _generate_filename(request, xform, remove_group_name=False, dataview_pk=False): + if request.GET.get("raw"): filename = None else: # append group name removed flag otherwise use the form id_string @@ -322,22 +352,24 @@ def _generate_filename(request, def _set_start_end_params(request, query): # check for start and end params - if 'start' in request.GET or 'end' in request.GET: - query = json.loads(query) \ - if isinstance(query, six.string_types) else query + if "start" in request.GET or "end" in request.GET: + query = json.loads(query) if isinstance(query, six.string_types) else query query[SUBMISSION_TIME] = {} try: - if request.GET.get('start'): - query[SUBMISSION_TIME]['$gte'] = _format_date_for_mongo( - request.GET['start'], datetime) - - if request.GET.get('end'): - query[SUBMISSION_TIME]['$lte'] = _format_date_for_mongo( - request.GET['end'], datetime) + if request.GET.get("start"): + query[SUBMISSION_TIME]["$gte"] = _format_date_for_mongo( + request.GET["start"], datetime + ) + + if request.GET.get("end"): + query[SUBMISSION_TIME]["$lte"] = _format_date_for_mongo( + request.GET["end"], datetime + ) except ValueError: raise exceptions.ParseError( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) else: query = json.dumps(query) @@ -348,16 +380,15 @@ def _get_extension_from_export_type(export_type): extension = export_type if export_type == Export.XLS_EXPORT: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" return extension def _format_date_for_mongo(x, datetime): # pylint: disable=W0621, C0103 - return datetime.strptime(x, - '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S') + return datetime.strptime(x, "%y_%m_%d_%H_%M_%S").strftime("%Y-%m-%dT%H:%M:%S") def process_async_export(request, xform, export_type, options=None): @@ -387,20 +418,23 @@ def process_async_export(request, xform, export_type, options=None): query = options.get("query") try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: payload = {"details": _("You don't have permission")} return Response( data=json.dumps(payload), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) else: if query: - options['query'] = query + options["query"] = query - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -408,25 +442,27 @@ def process_async_export(request, xform, export_type, options=None): if isinstance(credential, HttpResponseRedirect): return credential - options['google_credentials'] = credential.to_json() + options["google_credentials"] = credential.to_json() - if should_create_new_export(xform, export_type, options, request=request)\ - or export_type == Export.EXTERNAL_EXPORT: + if ( + should_create_new_export(xform, export_type, options, request=request) + or export_type == Export.EXTERNAL_EXPORT + ): resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, False, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, False, options=options + ) } else: - print('Do not create a new export.') + print("Do not create a new export.") export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, False, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, False, options=options + ) } else: resp = export_async_export_response(request, export) @@ -434,21 +470,19 @@ def process_async_export(request, xform, export_type, options=None): return resp -def _create_export_async(xform, - export_type, - query=None, - force_xlsx=False, - options=None): +def _create_export_async( + xform, export_type, query=None, force_xlsx=False, options=None +): + """ + Creates async exports + :param xform: + :param export_type: + :param query: + :param force_xlsx: + :param options: + :return: + job_uuid generated """ - Creates async exports - :param xform: - :param export_type: - :param query: - :param force_xlsx: - :param options: - :return: - job_uuid generated - """ export = check_pending_export(xform, export_type, options) if export: @@ -456,7 +490,8 @@ def _create_export_async(xform, try: export, async_result = viewer_task.create_async_export( - xform, export_type, query, force_xlsx, options=options) + xform, export_type, query, force_xlsx, options=options + ) except ExportConnectionError: raise ServiceUnavailable @@ -472,14 +507,16 @@ def export_async_export_response(request, export): """ if export.status == Export.SUCCESSFUL: if export.export_type not in [ - Export.EXTERNAL_EXPORT, Export.GOOGLE_SHEETS_EXPORT + Export.EXTERNAL_EXPORT, + Export.GOOGLE_SHEETS_EXPORT, ]: export_url = reverse( - 'export-detail', kwargs={'pk': export.pk}, request=request) + "export-detail", kwargs={"pk": export.pk}, request=request + ) else: export_url = export.export_url resp = async_status(SUCCESSFUL) - resp['export_url'] = export_url + resp["export_url"] = export_url elif export.status == Export.PENDING: resp = async_status(PENDING) else: @@ -492,13 +529,14 @@ def get_async_response(job_uuid, request, xform, count=0): """ Returns the status of an async task for the given job_uuid. """ + def _get_response(): export = get_object_or_404(Export, task_id=job_uuid) return export_async_export_response(request, export) try: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": resp = _get_response() else: resp = async_status(celery_state_to_status(job.state)) @@ -509,7 +547,7 @@ def _get_response(): if isinstance(result, dict): resp.update(result) else: - resp.update({'progress': str(result)}) + resp.update({"progress": str(result)}) except (OperationalError, ConnectionError) as e: report_exception("Connection Error", e, sys.exc_info()) if count > 0: @@ -518,7 +556,7 @@ def _get_response(): return get_async_response(job_uuid, request, xform, count + 1) except BacklogLimitExceeded: # most likely still processing - resp = async_status(celery_state_to_status('PENDING')) + resp = async_status(celery_state_to_status("PENDING")) return resp @@ -527,9 +565,9 @@ def response_for_format(data, format=None): # pylint: disable=W0622 """ Return appropriately formatted data in Response(). """ - if format == 'xml': + if format == "xml": formatted_data = data.xml - elif format == 'xls': + elif format == "xls": if not data.xls or not data.xls.storage.exists(data.xls.name): raise Http404() @@ -543,35 +581,38 @@ def generate_google_web_flow(request): """ Returns a OAuth2WebServerFlow object from the request redirect_uri. """ - if 'redirect_uri' in request.GET: - redirect_uri = request.GET.get('redirect_uri') - elif 'redirect_uri' in request.POST: - redirect_uri = request.POST.get('redirect_uri') - elif 'redirect_uri' in request.query_params: - redirect_uri = request.query_params.get('redirect_uri') - elif 'redirect_uri' in request.data: - redirect_uri = request.data.get('redirect_uri') + if "redirect_uri" in request.GET: + redirect_uri = request.GET.get("redirect_uri") + elif "redirect_uri" in request.POST: + redirect_uri = request.POST.get("redirect_uri") + elif "redirect_uri" in request.query_params: + redirect_uri = request.query_params.get("redirect_uri") + elif "redirect_uri" in request.data: + redirect_uri = request.data.get("redirect_uri") else: redirect_uri = settings.GOOGLE_STEP2_URI return OAuth2WebServerFlow( client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=' '.join([ - 'https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', - 'https://www.googleapis.com/auth/drive.file' - ]), + scope=" ".join( + [ + "https://docs.google.com/feeds/", + "https://spreadsheets.google.com/feeds/", + "https://www.googleapis.com/auth/drive.file", + ] + ), redirect_uri=redirect_uri, - prompt="consent") + prompt="consent", + ) def _get_google_credential(request): token = None credential = None if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') + storage = Storage(TokenStorageModel, "id", request.user, "credential") credential = storage.get() - elif request.session.get('access_token'): + elif request.session.get("access_token"): credential = google_client.OAuth2Credentials.from_json(token) if credential: diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 9507d44f25..f518e4b84c 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -191,7 +191,7 @@ "django.contrib.admindocs", "django.contrib.gis", "registration", - "django_nose", + # "django_nose", "django_digest", "corsheaders", "oauth2_provider", @@ -462,8 +462,8 @@ def configure_logging(logger, **kwargs): # default content length for submission requests DEFAULT_CONTENT_LENGTH = 10000000 -TEST_RUNNER = "django_nose.NoseTestSuiteRunner" -NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] +# TEST_RUNNER = "django_nose.NoseTestSuiteRunner" +# NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] # fake endpoints for testing TEST_HTTP_HOST = "testserver.com" From c17da0c80e062e411392ce212258f643b2d035d8 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:41:53 +0300 Subject: [PATCH 005/234] Disable SPSS export in the event of an ImportError exception. savReaderWriter needs to be updated to handle collections Iterable correctly. --- onadata/libs/utils/api_export_tools.py | 6 +++++- onadata/libs/utils/export_builder.py | 10 +++++++++- onadata/libs/utils/export_tools.py | 7 ++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index f52bc3b552..a271e47fad 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -27,7 +27,11 @@ from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse -from savReaderWriter import SPSSIOError + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.main.models import TokenStorageModel from onadata.apps.viewer import tasks as viewer_task diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 018b7f74ec..c54483a5e0 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -23,7 +23,11 @@ from openpyxl.workbook import Workbook from pyxform.question import Question from pyxform.section import RepeatingSection, Section -from savReaderWriter import SavWriter + +try: + from savReaderWriter import SavWriter +except ImportError: + SavWriter = None from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.logger.models.xform import ( @@ -1411,6 +1415,10 @@ def _check_sav_column(self, column, columns): return column def to_zipped_sav(self, path, data, *args, **kwargs): + if SavWriter is None: + # Fail silently + return + total_records = kwargs.get("total_records") def write_row(row, csv_writer, fields): diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index f4ba38bd7e..dc12d5ddee 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -26,7 +26,12 @@ from six import iteritems from json2xlsclient.client import Client from rest_framework import exceptions -from savReaderWriter import SPSSIOError + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception + from multidb.pinning import use_master from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm From d6fd696b2e3d904f20e4c031509292c327713b0a Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:48:46 +0300 Subject: [PATCH 006/234] django.utils.decorators.available_attrs() - This function returns functools.WRAPPER_ASSIGNMENTS Removed in Django 3.0 --- onadata/libs/renderers/renderers.py | 2 +- onadata/libs/utils/decorators.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index fd508a74d9..077c189135 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -15,7 +15,7 @@ from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator -from fsix import iteritems +from six import iteritems from rest_framework import negotiation from rest_framework.renderers import ( BaseRenderer, diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 0d556d8efe..55a0f6cc46 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -2,7 +2,6 @@ from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME -from django.utils.decorators import available_attrs from django.conf import settings from django.http import HttpResponseRedirect @@ -17,7 +16,7 @@ def with_check_obj(*args, **kwargs): def is_owner(view_func): - @wraps(view_func, assigned=available_attrs(view_func)) + @wraps(view_func) def _wrapped_view(request, *args, **kwargs): # assume username is first arg if request.user.is_authenticated: From 914f571d6d98b5bd175e7f743fde81ca27d4bf61 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:52:20 +0300 Subject: [PATCH 007/234] Disable SPSS export in the event of an ImportError exception. savReaderWriter needs to be updated to handle collections Iterable correctly. --- onadata/apps/viewer/views.py | 760 ++++++++++++++++++++--------------- 1 file changed, 441 insertions(+), 319 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 03e24606a4..b5d260846d 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -16,9 +16,13 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage, get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseNotFound, - HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, +) from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse @@ -26,9 +30,12 @@ from django.views.decorators.http import require_POST from dpath import util as dpath_util from oauth2client import client as google_client -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage -from savReaderWriter import SPSSIOError +from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.logger.models import Attachment from onadata.apps.logger.views import download_jsonform @@ -40,17 +47,30 @@ from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.chart_tools import build_chart_data from onadata.libs.utils.export_tools import ( - DEFAULT_GROUP_DELIMITER, generate_export, kml_export_data, - newest_export_for, should_create_new_export, str_to_bool) + DEFAULT_GROUP_DELIMITER, + generate_export, + kml_export_data, + newest_export_for, + should_create_new_export, + str_to_bool, +) from onadata.libs.utils.google import google_flow from onadata.libs.utils.image_tools import image_url from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import ( - generate_content_disposition_header, response_with_mimetype_and_name) -from onadata.libs.utils.user_auth import (get_xform_and_perms, has_permission, - helper_auth_helper) + generate_content_disposition_header, + response_with_mimetype_and_name, +) +from onadata.libs.utils.user_auth import ( + get_xform_and_perms, + has_permission, + helper_auth_helper, +) from onadata.libs.utils.viewer_tools import ( - create_attachments_zipfile, export_def_from_filename, get_form) + create_attachments_zipfile, + export_def_from_filename, + get_form, +) from onadata.libs.utils.common_tools import get_uuid @@ -58,15 +78,18 @@ def _get_start_end_submission_time(request): start = None end = None try: - if request.GET.get('start'): - start = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['start'], '%y_%m_%d_%H_%M_%S')) - if request.GET.get('end'): - end = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['end'], '%y_%m_%d_%H_%M_%S')) + if request.GET.get("start"): + start = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["start"], "%y_%m_%d_%H_%M_%S") + ) + if request.GET.get("end"): + end = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S") + ) except ValueError: return HttpResponseBadRequest( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) return start, end @@ -75,16 +98,16 @@ def encode(time_str): """ Reformat a time string into YYYY-MM-dd HH:mm:ss. """ - return strftime("%Y-%m-%d %H:%M:%S", - strptime(time_str, "%Y_%m_%d_%H_%M_%S")) + return strftime("%Y-%m-%d %H:%M:%S", strptime(time_str, "%Y_%m_%d_%H_%M_%S")) def format_date_for_mongo(time_str): """ Reformat a time string into YYYY-MM-ddTHH:mm:ss. """ - return datetime.strptime(time_str, '%y_%m_%d_%H_%M_%S')\ - .strftime('%Y-%m-%dT%H:%M:%S') + return datetime.strptime(time_str, "%y_%m_%d_%H_%M_%S").strftime( + "%Y-%m-%dT%H:%M:%S" + ) def instances_for_export(data_dictionary, start=None, end=None): @@ -93,9 +116,9 @@ def instances_for_export(data_dictionary, start=None, end=None): """ kwargs = dict() if start: - kwargs['date_created__gte'] = start + kwargs["date_created__gte"] = start if end: - kwargs['date_created__lte'] = end + kwargs["date_created__lte"] = end return data_dictionary.instances.filter(**kwargs) @@ -108,11 +131,9 @@ def set_instances_for_export(id_string, owner, request): is successful or not respectively. """ data_dictionary = get_object_or_404( - DataDictionary, - id_string__iexact=id_string, - user=owner, - deletd_at__isnull=True) - start, end = request.GET.get('start'), request.GET.get('end') + DataDictionary, id_string__iexact=id_string, user=owner, deletd_at__isnull=True + ) + start, end = request.GET.get("start"), request.GET.get("end") if start: try: start = encode(start) @@ -121,7 +142,8 @@ def set_instances_for_export(id_string, owner, request): return [ False, HttpResponseBadRequest( - _(u'Start time format must be YY_MM_DD_hh_mm_ss')) + _("Start time format must be YY_MM_DD_hh_mm_ss") + ), ] if end: try: @@ -130,12 +152,12 @@ def set_instances_for_export(id_string, owner, request): # bad format return [ False, - HttpResponseBadRequest( - _(u'End time format must be YY_MM_DD_hh_mm_ss')) + HttpResponseBadRequest(_("End time format must be YY_MM_DD_hh_mm_ss")), ] if start or end: data_dictionary.instances_for_export = instances_for_export( - data_dictionary, start, end) + data_dictionary, start, end + ) return [True, data_dictionary] @@ -147,48 +169,48 @@ def average(values): return sum(values, 0.0) / len(values) if values else None -def map_view(request, username, id_string, template='map.html'): +def map_view(request, username, id_string, template="map.html"): """ Map view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'content_user': owner, 'xform': xform} - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - - data['form_view'] = True - data['jsonform_url'] = reverse( - download_jsonform, - kwargs={"username": username, - "id_string": id_string}) - data['enketo_edit_url'] = reverse( - 'edit_data', - kwargs={"username": username, - "id_string": id_string, - "data_id": 0}) - data['enketo_add_url'] = reverse( - 'enter_data', kwargs={"username": username, - "id_string": id_string}) - - data['enketo_add_with_url'] = reverse( - 'add_submission_with', - kwargs={"username": username, - "id_string": id_string}) - data['mongo_api_url'] = reverse( - 'mongo_view_api', - kwargs={"username": username, - "id_string": id_string}) - data['delete_data_url'] = reverse( - 'delete_data', kwargs={"username": username, - "id_string": id_string}) - data['mapbox_layer'] = MetaData.mapbox_layer_upload(xform) + return HttpResponseForbidden(_("Not shared.")) + data = {"content_user": owner, "xform": xform} + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + + data["form_view"] = True + data["jsonform_url"] = reverse( + download_jsonform, kwargs={"username": username, "id_string": id_string} + ) + data["enketo_edit_url"] = reverse( + "edit_data", kwargs={"username": username, "id_string": id_string, "data_id": 0} + ) + data["enketo_add_url"] = reverse( + "enter_data", kwargs={"username": username, "id_string": id_string} + ) + + data["enketo_add_with_url"] = reverse( + "add_submission_with", kwargs={"username": username, "id_string": id_string} + ) + data["mongo_api_url"] = reverse( + "mongo_view_api", kwargs={"username": username, "id_string": id_string} + ) + data["delete_data_url"] = reverse( + "delete_data", kwargs={"username": username, "id_string": id_string} + ) + data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) audit = {"xform": xform.id_string} - audit_log(Actions.FORM_MAP_VIEWED, request.user, owner, - _("Requested map on '%(id_string)s'.") % - {'id_string': xform.id_string}, audit, request) + audit_log( + Actions.FORM_MAP_VIEWED, + request.user, + owner, + _("Requested map on '%(id_string)s'.") % {"id_string": xform.id_string}, + audit, + request, + ) return render(request, template, data) @@ -196,7 +218,7 @@ def map_embed_view(request, username, id_string): """ Embeded map view. """ - return map_view(request, username, id_string, template='map_embed.html') + return map_view(request, username, id_string, template="map_embed.html") def add_submission_with(request, username, id_string): @@ -209,51 +231,51 @@ def geopoint_xpaths(username, id_string): Returns xpaths with elements of type 'geopoint'. """ data_dictionary = DataDictionary.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) return [ e.get_abbreviated_xpath() for e in data_dictionary.get_survey_elements() - if e.bind.get(u'type') == u'geopoint' + if e.bind.get("type") == "geopoint" ] - value = request.GET.get('coordinates') + value = request.GET.get("coordinates") xpaths = geopoint_xpaths(username, id_string) xml_dict = {} for path in xpaths: dpath_util.new(xml_dict, path, value) context = { - 'username': username, - 'id_string': id_string, - 'xml_content': dict2xml(xml_dict) + "username": username, + "id_string": id_string, + "xml_content": dict2xml(xml_dict), } - instance_xml = loader.get_template("instance_add.xml")\ - .render(context) + instance_xml = loader.get_template("instance_add.xml").render(context) url = settings.ENKETO_API_INSTANCE_IFRAME_URL return_url = reverse( - 'thank_you_submission', - kwargs={"username": username, - "id_string": id_string}) + "thank_you_submission", kwargs={"username": username, "id_string": id_string} + ) if settings.DEBUG: openrosa_url = "https://dev.formhub.org/{}".format(username) else: openrosa_url = request.build_absolute_uri("/{}".format(username)) payload = { - 'return_url': return_url, - 'form_id': id_string, - 'server_url': openrosa_url, - 'instance': instance_xml, - 'instance_id': get_uuid() + "return_url": return_url, + "form_id": id_string, + "server_url": openrosa_url, + "instance": instance_xml, + "instance_id": get_uuid(), } response = requests.post( url, data=payload, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) - return HttpResponse(response.text, content_type='application/json') + return HttpResponse(response.text, content_type="application/json") # pylint: disable=W0613 @@ -270,35 +292,35 @@ def data_export(request, username, id_string, export_type): Data export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) query = request.GET.get("query") extension = export_type # check if we should force xlsx - force_xlsx = request.GET.get('xls') != 'true' + force_xlsx = request.GET.get("xls") != "true" if export_type == Export.XLS_EXPORT and force_xlsx: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" audit = {"xform": xform.id_string, "export_type": export_type} - options = { - "extension": extension, - "username": username, - "id_string": id_string - } + options = {"extension": extension, "username": username, "id_string": id_string} if query: - options['query'] = query + options["query"] = query # check if we need to re-generate, # we always re-generate if a filter is specified - if should_create_new_export(xform, export_type, options) or query or\ - 'start' in request.GET or 'end' in request.GET: + if ( + should_create_new_export(xform, export_type, options) + or query + or "start" in request.GET + or "end" in request.GET + ): # check for start and end params start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) @@ -306,11 +328,14 @@ def data_export(request, username, id_string, export_type): try: export = generate_export(export_type, xform, None, options) audit_log( - Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - 'export_type': export_type.upper() - }, audit, request) + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) except SPSSIOError as e: @@ -320,10 +345,14 @@ def data_export(request, username, id_string, export_type): # log download as well audit_log( - Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) if not export.filename and not export.error_message: # tends to happen when using newset_export_for. @@ -335,14 +364,15 @@ def data_export(request, username, id_string, export_type): # xlsx if it exceeds limits __, extension = os.path.splitext(export.filename) extension = extension[1:] - if request.GET.get('raw'): + if request.GET.get("raw"): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[extension], id_string, extension=extension, - file_path=export.filepath) + file_path=export.filepath, + ) return response @@ -355,15 +385,15 @@ def create_export(request, username, id_string, export_type): Create async export tasks view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) credential = None if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -373,69 +403,78 @@ def create_export(request, username, id_string, export_type): return credential query = request.POST.get("query") - force_xlsx = request.POST.get('xls') != 'true' + force_xlsx = request.POST.get("xls") != "true" # export options - group_delimiter = request.POST.get("options[group_delimiter]", '/') - if group_delimiter not in ['.', '/']: + group_delimiter = request.POST.get("options[group_delimiter]", "/") + if group_delimiter not in [".", "/"]: return HttpResponseBadRequest( - _("%s is not a valid delimiter" % group_delimiter)) + _("%s is not a valid delimiter" % group_delimiter) + ) # default is True, so when dont_.. is yes # split_select_multiples becomes False - split_select_multiples = request.POST.get( - "options[dont_split_select_multiples]", "no") == "no" + split_select_multiples = ( + request.POST.get("options[dont_split_select_multiples]", "no") == "no" + ) - binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES', - False) + binary_select_multiples = getattr(settings, "BINARY_SELECT_MULTIPLES", False) remove_group_name = request.POST.get("options[remove_group_name]", "false") value_select_multiples = request.POST.get( - "options[value_select_multiples]", "false") + "options[value_select_multiples]", "false" + ) # external export option meta = request.POST.get("meta") options = { - 'group_delimiter': group_delimiter, - 'split_select_multiples': split_select_multiples, - 'binary_select_multiples': binary_select_multiples, - 'value_select_multiples': str_to_bool(value_select_multiples), - 'remove_group_name': str_to_bool(remove_group_name), - 'meta': meta.replace(",", "") if meta else None, - 'google_credentials': credential + "group_delimiter": group_delimiter, + "split_select_multiples": split_select_multiples, + "binary_select_multiples": binary_select_multiples, + "value_select_multiples": str_to_bool(value_select_multiples), + "remove_group_name": str_to_bool(remove_group_name), + "meta": meta.replace(",", "") if meta else None, + "google_credentials": credential, } try: create_async_export(xform, export_type, query, force_xlsx, options) except ExportTypeError: - return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + return HttpResponseBadRequest(_("%s is not a valid export type" % export_type)) else: audit = {"xform": xform.id_string, "export_type": export_type} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'export_type': export_type.upper(), - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created %(export_type)s export on '%(id_string)s'.") + % { + "export_type": export_type.upper(), + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def _get_google_credential(request): token = None if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') + storage = Storage(TokenStorageModel, "id", request.user, "credential") credential = storage.get() - elif request.session.get('access_token'): + elif request.session.get("access_token"): credential = google_client.OAuth2Credentials.from_json(token) - return credential or HttpResponseRedirect( - google_flow.step1_get_authorize_url()) + return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) def export_list(request, username, id_string, export_type): @@ -446,7 +485,8 @@ def export_list(request, username, id_string, export_type): if export_type not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( - _(u'Export type "%s" is not supported.' % export_type)) + _('Export type "%s" is not supported.' % export_type) + ) if export_type == Export.GOOGLE_SHEETS_EXPORT: # Retrieve google creds or redirect to google authorization page @@ -454,58 +494,57 @@ def export_list(request, username, id_string, export_type): if isinstance(credential, HttpResponseRedirect): return credential owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) # Get meta and token - export_token = request.GET.get('token') - export_meta = request.GET.get('meta') + export_token = request.GET.get("token") + export_meta = request.GET.get("meta") options = { - 'group_delimiter': DEFAULT_GROUP_DELIMITER, - 'remove_group_name': False, - 'split_select_multiples': True, - 'binary_select_multiples': False, - 'meta': export_meta, - 'token': export_token, - 'google_credentials': credential + "group_delimiter": DEFAULT_GROUP_DELIMITER, + "remove_group_name": False, + "split_select_multiples": True, + "binary_select_multiples": False, + "meta": export_meta, + "token": export_token, + "google_credentials": credential, } if should_create_new_export(xform, export_type, options): try: create_async_export( - xform, - export_type, - query=None, - force_xlsx=True, - options=options) + xform, export_type, query=None, force_xlsx=True, options=options + ) except Export.ExportTypeError: return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + _("%s is not a valid export type" % export_type) + ) - metadata_qs = MetaData.objects.filter(object_id=xform.id, - data_type="external_export")\ - .values('id', 'data_value') + metadata_qs = MetaData.objects.filter( + object_id=xform.id, data_type="external_export" + ).values("id", "data_value") for metadata in metadata_qs: - metadata['data_value'] = metadata.get('data_value').split('|')[0] + metadata["data_value"] = metadata.get("data_value").split("|")[0] data = { - 'username': owner.username, - 'xform': xform, - 'export_type': export_type, - 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], - 'exports': Export.objects.filter( - xform=xform, export_type=export_type).order_by('-created_on'), - 'metas': metadata_qs + "username": owner.username, + "xform": xform, + "export_type": export_type, + "export_type_name": Export.EXPORT_TYPE_DICT[export_type], + "exports": Export.objects.filter(xform=xform, export_type=export_type).order_by( + "-created_on" + ), + "metas": metadata_qs, } # yapf: disable - return render(request, 'export_list.html', data) + return render(request, "export_list.html", data) def export_progress(request, username, id_string, export_type): @@ -513,47 +552,52 @@ def export_progress(request, username, id_string, export_type): Async export progress view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db - export_ids = request.GET.getlist('export_ids') + export_ids = request.GET.getlist("export_ids") exports = Export.objects.filter( - xform=xform, id__in=export_ids, export_type=export_type) + xform=xform, id__in=export_ids, export_type=export_type + ) statuses = [] for export in exports: status = { - 'complete': False, - 'url': None, - 'filename': None, - 'export_id': export.id + "complete": False, + "url": None, + "filename": None, + "export_id": export.id, } if export.status == Export.SUCCESSFUL: - status['url'] = reverse( + status["url"] = reverse( export_download, kwargs={ - 'username': owner.username, - 'id_string': xform.id_string, - 'export_type': export.export_type, - 'filename': export.filename - }) - status['filename'] = export.filename - if export.export_type == Export.GOOGLE_SHEETS_EXPORT and \ - export.export_url is None: - status['url'] = None - if export.export_type == Export.EXTERNAL_EXPORT \ - and export.export_url is None: - status['url'] = None + "username": owner.username, + "id_string": xform.id_string, + "export_type": export.export_type, + "filename": export.filename, + }, + ) + status["filename"] = export.filename + if ( + export.export_type == Export.GOOGLE_SHEETS_EXPORT + and export.export_url is None + ): + status["url"] = None + if ( + export.export_type == Export.EXTERNAL_EXPORT + and export.export_url is None + ): + status["url"] = None # mark as complete if it either failed or succeeded but NOT pending - if export.status == Export.SUCCESSFUL \ - or export.status == Export.FAILED: - status['complete'] = True + if export.status == Export.SUCCESSFUL or export.status == Export.FAILED: + status["complete"] = True statuses.append(status) - return HttpResponse(json.dumps(statuses), content_type='application/json') + return HttpResponse(json.dumps(statuses), content_type="application/json") def export_download(request, username, id_string, export_type, filename): @@ -561,31 +605,38 @@ def export_download(request, username, id_string, export_type, filename): Export download view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db export = get_object_or_404(Export, xform=xform, filename=filename) - if (export_type == Export.GOOGLE_SHEETS_EXPORT or - export_type == Export.EXTERNAL_EXPORT) and \ - export.export_url is not None: + if ( + export_type == Export.GOOGLE_SHEETS_EXPORT + or export_type == Export.EXTERNAL_EXPORT + ) and export.export_url is not None: return HttpResponseRedirect(export.export_url) ext, mime_type = export_def_from_filename(export.filename) audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export '%(filename)s' " - "on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded %(export_type)s export '%(filename)s' " "on '%(id_string)s'.") + % { + "export_type": export.export_type.upper(), + "filename": export.filename, + "id_string": xform.id_string, + }, + audit, + request, + ) + if request.GET.get("raw"): id_string = None default_storage = get_storage_class()() @@ -597,7 +648,8 @@ def export_download(request, username, id_string, export_type, filename): name=basename, extension=ext, file_path=export.filepath, - show_date=False) + show_date=False, + ) return response @@ -608,33 +660,41 @@ def delete_export(request, username, id_string, export_type): Delete export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - export_id = request.POST.get('export_id') + export_id = request.POST.get("export_id") # find the export entry in the db export = get_object_or_404(Export, id=export_id) export.delete() audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Deleted %(export_type)s export '%(filename)s'" - " on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Deleted %(export_type)s export '%(filename)s'" " on '%(id_string)s'.") + % { + "export_type": export.export_type.upper(), + "filename": export.filename, + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def zip_export(request, username, id_string): @@ -642,12 +702,12 @@ def zip_export(request, username, id_string): Zip export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - if request.GET.get('raw'): + return HttpResponseForbidden(_("Not shared.")) + if request.GET.get("raw"): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) @@ -656,21 +716,35 @@ def zip_export(request, username, id_string): try: zip_file = create_attachments_zipfile(attachments) audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created ZIP export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded ZIP export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + if request.GET.get("raw"): id_string = None - response = response_with_mimetype_and_name('zip', id_string) + response = response_with_mimetype_and_name("zip", id_string) response.write(FileWrapper(zip_file)) - response['Content-Length'] = zip_file.tell() + response["Content-Length"] = zip_file.tell() zip_file.seek(0) finally: if zip_file: @@ -685,29 +759,42 @@ def kml_export(request, username, id_string): """ # read the locations from the database owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'data': kml_export_data(id_string, user=owner, xform=xform)} + return HttpResponseForbidden(_("Not shared.")) + data = {"data": kml_export_data(id_string, user=owner, xform=xform)} response = render( - request, - "survey.kml", - data, - content_type="application/vnd.google-earth.kml+xml") - response['Content-Disposition'] = \ - generate_content_disposition_header(id_string, 'kml') + request, "survey.kml", data, content_type="application/vnd.google-earth.kml+xml" + ) + response["Content-Disposition"] = generate_content_disposition_header( + id_string, "kml" + ) audit = {"xform": xform.id_string, "export_type": Export.KML_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created KML export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded KML export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return response @@ -725,24 +812,22 @@ def google_xls_export(request, username, id_string): pass else: token = token_storage.token - elif request.session.get('access_token'): - token = request.session.get('access_token') + elif request.session.get("access_token"): + token = request.session.get("access_token") if token is None: request.session["google_redirect_url"] = reverse( - google_xls_export, - kwargs={'username': username, - 'id_string': id_string}) + google_xls_export, kwargs={"username": username, "id_string": id_string} + ) return HttpResponseRedirect(google_flow.step1_get_authorize_url()) owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - is_valid, data_dictionary = set_instances_for_export( - id_string, owner, request) + is_valid, data_dictionary = set_instances_for_export(id_string, owner, request) if not is_valid: return data_dictionary @@ -755,10 +840,17 @@ def google_xls_export(request, username, id_string): url = None os.unlink(tmp.name) audit = {"xform": xform.id_string, "export_type": "google"} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created Google Docs export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created Google Docs export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect(url) @@ -768,51 +860,59 @@ def data_view(request, username, id_string): Data view displays submission data. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'id_string__iexact': id_string, 'user': owner}) + xform = get_form({"id_string__iexact": id_string, "user": owner}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - data = {'owner': owner, 'xform': xform} + data = {"owner": owner, "xform": xform} audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, owner, - _("Requested data view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + owner, + _("Requested data view for '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return render(request, "data_view.html", data) -def attachment_url(request, size='medium'): +def attachment_url(request, size="medium"): """ Redirects to image attachment of the specified size, defaults to 'medium'. """ - media_file = request.GET.get('media_file') - no_redirect = request.GET.get('no_redirect') + media_file = request.GET.get("media_file") + no_redirect = request.GET.get("no_redirect") if not media_file: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) result = Attachment.objects.filter(media_file=media_file).order_by()[0:1] if not result: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) attachment = result[0] - if size == 'original' and no_redirect == 'true': + if size == "original" and no_redirect == "true": response = response_with_mimetype_and_name( attachment.mimetype, attachment.name, extension=attachment.extension, - file_path=attachment.media_file.name) + file_path=attachment.media_file.name, + ) return response - if not attachment.mimetype.startswith('image'): + if not attachment.mimetype.startswith("image"): return redirect(attachment.media_file.url) media_url = image_url(attachment, size) if media_url: return redirect(media_url) - return HttpResponseNotFound(_(u'Error: Attachment not found')) + return HttpResponseNotFound(_("Error: Attachment not found")) def instance(request, username, id_string): @@ -821,26 +921,41 @@ def instance(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, xform.user, - _("Requested instance view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + xform.user, + _("Requested instance view for '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) - return render(request, 'instance.html', { - 'username': username, - 'id_string': id_string, - 'xform': xform, - 'can_edit': can_edit - }) + return render( + request, + "instance.html", + { + "username": username, + "id_string": id_string, + "xform": xform, + "can_edit": can_edit, + }, + ) def charts(request, username, id_string): @@ -849,20 +964,24 @@ def charts(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) try: - lang_index = int(request.GET.get('lang', 0)) + lang_index = int(request.GET.get("lang", 0)) except ValueError: lang_index = 0 try: - page = int(request.GET.get('page', 0)) + page = int(request.GET.get("page", 0)) except ValueError: page = 0 else: @@ -871,14 +990,13 @@ def charts(request, username, id_string): summaries = build_chart_data(xform, lang_index, page) if request.is_ajax(): - template = 'charts_snippet.html' + template = "charts_snippet.html" else: - template = 'charts.html' + template = "charts.html" - return render(request, template, - {'xform': xform, - 'summaries': summaries, - 'page': page + 1}) + return render( + request, template, {"xform": xform, "summaries": summaries, "page": page + 1} + ) def stats_tables(request, username, id_string): @@ -887,10 +1005,14 @@ def stats_tables(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) - - return render(request, 'stats_tables.html', {'xform': xform}) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) + + return render(request, "stats_tables.html", {"xform": xform}) From 49dea8ce118d73aa2ea05db55a724565e8336f5f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 19 Apr 2022 16:54:31 +0300 Subject: [PATCH 008/234] Upgrade dependencies for Django 3.x upgrade Also converted to using pyproject.toml --- .gitignore | 2 + pyproject.toml | 8 ++ requirements/base.in | 14 +- requirements/base.pip | 313 +++++++++++++----------------------------- setup.cfg | 112 +++++++++++++++ setup.py | 120 +--------------- 6 files changed, 225 insertions(+), 344 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index d58bfe49db..0c8536fb7c 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,5 @@ tags .bash_history .inputrc + +.eggs diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..fdad2c0aba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools >= 48", + "setuptools_scm[toml] >= 4, <6", + "setuptools_scm_git_archive", + "wheel >= 0.29.0", +] +build-backend = 'setuptools.build_meta' diff --git a/requirements/base.in b/requirements/base.in index 9c74c25087..ee5efd28bc 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -3,10 +3,10 @@ # installed from Git -e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client --e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc +#-e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest +#-e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest +#-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router +#-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip +#-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient +#-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client +#-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index d3688c9e78..9a694656b8 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -1,90 +1,64 @@ # -# This file is autogenerated by pip-compile with python 3.6 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile --output-file=requirements/base.pip requirements/base.in # -e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform --e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest - # via -r requirements/base.in --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router - # via -r requirements/base.in --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client - # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc - # via -r requirements/base.in --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip - # via -r requirements/base.in --e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest - # via -r requirements/base.in --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient - # via -r requirements/base.in alabaster==0.7.12 # via sphinx -amqp==5.0.6 +amqp==5.1.1 # via kombu -analytics-python==1.3.1 +analytics-python==1.4.0 # via onadata appoptics-metrics==5.1.0 # via onadata -argparse==1.4.0 - # via unittest2 -attrs==21.2.0 - # via jsonschema +asgiref==3.5.0 + # via django +async-timeout==4.0.2 + # via redis babel==2.9.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery -boto3==1.17.74 - # via tabulator -botocore==1.20.74 - # via - # boto3 - # s3transfer -cached-property==1.5.2 - # via tableschema -celery==5.0.5 +celery==5.2.6 # via onadata -certifi==2020.12.5 +certifi==2021.10.8 # via requests -cffi==1.14.5 +cffi==1.15.0 # via cryptography -chardet==4.0.0 - # via - # datapackage - # requests - # tabulator -click==7.1.2 +charset-normalizer==2.0.12 + # via requests +click==8.1.2 # via # celery # click-didyoumean # click-plugins # click-repl - # datapackage - # tableschema - # tabulator -click-didyoumean==0.0.3 +click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.1.6 +click-repl==0.2.0 # via celery -cryptography==3.4.7 +cryptography==36.0.2 # via # jwcrypto # onadata - # pyjwt -datapackage==1.15.2 - # via pyfloip defusedxml==0.7.1 - # via djangorestframework-xml -deprecated==1.2.12 - # via onadata -dict2xml==1.7.0 + # via + # djangorestframework-xml + # pyxform +deprecated==1.2.13 + # via + # jwcrypto + # onadata + # redis +dict2xml==1.7.1 # via onadata -django==2.2.23 +django==3.2.13 # via # django-cors-headers # django-debug-toolbar @@ -99,56 +73,53 @@ django==2.2.23 # djangorestframework # djangorestframework-guardian # djangorestframework-jsonapi - # jsonfield - # ona-oidc # onadata -django-activity-stream==0.10.0 +django-activity-stream==1.4.0 # via onadata -django-cors-headers==3.7.0 +django-cors-headers==3.11.0 # via onadata -django-debug-toolbar==3.2.1 +django-debug-toolbar==3.2.4 # via onadata -django-filter==2.4.0 +django-filter==21.1 # via onadata -django-guardian==2.3.0 +django-guardian==2.4.0 # via # djangorestframework-guardian # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==1.5.0 +django-oauth-toolkit==1.7.1 # via onadata -django-ordered-model==3.4.3 +django-ordered-model==3.5 # via onadata django-query-builder==2.0.1 # via onadata -django-redis==5.0.0 +django-redis==5.2.0 # via onadata -django-registration-redux==2.9 +django-registration-redux==2.10 # via onadata -django-render-block==0.8.1 +django-render-block==0.9.1 # via django-templated-email -django-reversion==3.0.9 +django-reversion==5.0.0 # via onadata -django-taggit==1.4.0 +django-taggit==2.1.0 # via onadata -django-templated-email==2.3.0 +django-templated-email==3.0.0 # via onadata -djangorestframework==3.12.4 +djangorestframework==3.13.1 # via # djangorestframework-csv # djangorestframework-gis # djangorestframework-guardian # djangorestframework-jsonapi - # ona-oidc # onadata djangorestframework-csv==2.1.1 # via onadata -djangorestframework-gis==0.17 +djangorestframework-gis==0.18 # via onadata djangorestframework-guardian==0.3.0 # via onadata -djangorestframework-jsonapi==4.2.0 +djangorestframework-jsonapi==5.0.0 # via onadata djangorestframework-jsonp==1.0.2 # via onadata @@ -156,82 +127,42 @@ djangorestframework-xml==2.0.0 # via onadata docutils==0.17.1 # via sphinx -dpath==2.0.1 +dpath==2.0.6 # via onadata elaphe3==0.2.0 # via onadata et-xmlfile==1.1.0 # via openpyxl -flake8==3.9.2 +flake8==4.0.1 # via onadata -fleming==0.6.0 +fleming==0.7.0 # via django-query-builder -formencode==2.0.1 - # via pyxform -future==0.18.2 - # via python-json2xlsclient geojson==2.5.0 # via onadata -greenlet==1.1.0 - # via sqlalchemy httmock==1.4.0 # via onadata -httplib2==0.19.1 - # via - # oauth2client - # onadata -idna==2.10 +httplib2==0.20.4 + # via onadata +idna==3.3 # via requests -ijson==3.1.4 - # via tabulator -imagesize==1.2.0 +imagesize==1.3.0 # via sphinx -importlib-metadata==4.8.1 - # via - # flake8 - # jsonpickle - # jsonschema - # kombu - # markdown - # sqlalchemy inflection==0.5.1 # via djangorestframework-jsonapi -isodate==0.6.0 - # via tableschema -jinja2==2.11.3 +jinja2==3.1.1 # via sphinx -jmespath==0.10.0 - # via - # boto3 - # botocore -jsonfield==0.9.23 - # via onadata -jsonlines==2.0.0 - # via tabulator -jsonpickle==2.0.0 +jsonpickle==2.1.0 # via onadata -jsonpointer==2.1 - # via datapackage -jsonschema==3.2.0 - # via - # datapackage - # tableschema -jwcrypto==0.8 +jwcrypto==1.0 # via django-oauth-toolkit -kombu==5.0.2 +kombu==5.2.4 # via celery -linear-tsv==1.1.0 - # via tabulator -linecache2==1.0.0 - # via traceback2 -lxml==4.6.3 +lxml==4.8.0 # via onadata -markdown==3.3.4 +markdown==3.3.6 # via onadata -markupsafe==1.1.1 - # via - # jinja2 - # sphinx +markupsafe==2.1.1 + # via jinja2 mccabe==0.6.1 # via flake8 mock==4.0.3 @@ -242,190 +173,132 @@ monotonic==1.6 # via analytics-python nose==1.3.7 # via django-nose -numpy==1.19.5 +numpy==1.22.3 # via onadata -oauthlib==3.1.0 +oauthlib==3.2.0 # via django-oauth-toolkit openpyxl==3.0.9 # via # onadata - # tabulator -packaging==20.9 - # via sphinx -paho-mqtt==1.5.1 + # pyxform +packaging==21.3 + # via + # redis + # sphinx +paho-mqtt==1.6.1 # via onadata -pillow==8.2.0 +pillow==9.1.0 # via # elaphe3 # onadata -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 - # via onadata -pyasn1==0.4.8 - # via - # oauth2client - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via oauth2client -pycodestyle==2.7.0 +pycodestyle==2.8.0 # via flake8 -pycparser==2.20 +pycparser==2.21 # via cffi -pyflakes==2.3.1 +pyflakes==2.4.0 # via flake8 -pygments==2.9.0 +pygments==2.11.2 # via sphinx -pyjwt[crypto]==2.1.0 - # via - # ona-oidc - # onadata +pyjwt==2.3.0 + # via onadata pylibmc==1.6.1 # via onadata -pymongo==3.11.4 +pymongo==4.1.1 # via onadata -pyparsing==2.4.7 +pyparsing==3.0.8 # via # httplib2 # packaging -pyrsistent==0.17.3 - # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # analytics-python - # botocore # fleming # onadata - # tableschema python-memcached==1.59 # via onadata -pytz==2021.1 +pytz==2022.1 # via # babel # celery # django # django-query-builder + # djangorestframework # fleming # onadata raven==6.10.0 # via onadata recaptcha-client==1.0.6 # via onadata -redis==3.5.3 +redis==4.2.2 # via django-redis -requests==2.25.1 +requests==2.27.1 # via # analytics-python - # datapackage # django-oauth-toolkit # httmock - # ona-oidc # onadata - # python-json2xlsclient # requests-mock # sphinx - # tableschema - # tabulator -requests-mock==1.9.2 - # via onadata -rfc3986==1.5.0 - # via tableschema -rsa==4.7.2 - # via oauth2client -s3transfer==0.4.2 - # via boto3 +requests-mock==1.9.3 + # via onadata savreaderwriter==3.4.2 # via onadata -simplejson==3.17.2 +simplejson==3.17.6 # via onadata six==1.16.0 # via # analytics-python # appoptics-metrics # click-repl - # datapackage - # django-oauth-toolkit # django-query-builder - # django-templated-email # djangorestframework-csv - # isodate - # jsonschema - # linear-tsv - # oauth2client # python-dateutil # python-memcached # requests-mock - # tableschema - # tabulator - # unittest2 -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx -sphinx==4.0.1 +sphinx==4.5.0 # via onadata sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy==1.4.15 - # via tabulator -sqlparse==0.4.1 +sqlparse==0.4.2 # via # django # django-debug-toolbar -tableschema==1.20.2 - # via datapackage -tabulator==1.53.5 - # via - # datapackage - # tableschema -traceback2==1.4.0 - # via unittest2 -typing-extensions==3.10.0.2 - # via importlib-metadata unicodecsv==0.14.1 # via - # datapackage # djangorestframework-csv # onadata - # pyxform - # tableschema - # tabulator -unittest2==1.1.0 - # via pyxform -urllib3==1.26.4 - # via - # botocore - # requests -uwsgi==2.0.19.1 +urllib3==1.26.9 + # via requests +uwsgi==2.0.20 # via onadata vine==5.0.0 # via # amqp # celery + # kombu wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.12.1 +wrapt==1.14.0 # via deprecated xlrd==2.0.1 # via # onadata # pyxform - # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 # via onadata -zipp==3.6.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..1b794f9ed5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,112 @@ +[metadata] +name = onadata +description = Collect Analyze and Share Data +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://github.com/onaio/onadata +author = Ona Systems Inc +author_email = support@ona.io +license = Copyright (c) 2022 Ona Systems Inc All rights reserved +license_file = LICENSE +classifiers = + Development Status :: 5 - Production/Stable + Programming Language :: Python :: 3.8 +project_urls = + Documentation = https://api.ona.io/api + Source = https://github.com/onaio/onadata + Tracker = https://github.com/onaio/onadata/issues + +[options] +packages = find: +platforms = any +install_requires = + Django>=2.2.20,<4 + django-guardian + django-registration-redux + django-templated-email + django-reversion + django-filter + django-nose + django-ordered-model + #generic relation + django-query-builder + celery + #cors + django-cors-headers + django-debug-toolbar + #oauth2 support + django-oauth-toolkit + #oauth2client + jsonpickle + #jwt + PyJWT + #captcha + recaptcha-client + #API support + djangorestframework + djangorestframework-csv + djangorestframework-gis + djangorestframework-guardian + djangorestframework-jsonapi + djangorestframework-jsonp + djangorestframework-xml + #geojson + geojson + #tagging + django-taggit + #database + #psycopg2-binary>2.7.1 + pymongo + #sms support + dict2xml + lxml + #pyxform + pyxform + #spss + savreaderwriter + #tests + mock + httmock + #memcached support + pylibmc + python-memcached + #XML Instance API utility + xmltodict + #docs + sphinx + Markdown + #others + unicodecsv + xlrd + xlwt + openpyxl + dpath + elaphe3 + httplib2 + modilabs-python-utils + numpy + Pillow + python-dateutil + pytz + requests + requests-mock + simplejson + uwsgi + flake8 + raven + django-activity-stream + paho-mqtt + cryptography + #Monitoring + analytics-python + appoptics-metrics + # Deprecation tagging + deprecated + # Redis cache + django-redis +python_requires = >= 3.8 +setup_requires = + setuptools_scm + +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index cd762c2947..0c93a07ecf 100644 --- a/setup.py +++ b/setup.py @@ -14,121 +14,7 @@ https://ona.io https://opendatakit.org """ +import setuptools -from setuptools import setup, find_packages - -import onadata - -setup( - name="onadata", - version=onadata.__version__, - description="Collect Analyze and Share Data!", - author="Ona Systems Inc", - author_email="support@ona.io", - license="Copyright (c) 2014 Ona Systems Inc All rights reserved.", - project_urls={ - 'Source': 'https://github.com/onaio/onadata', - }, - packages=find_packages(exclude=['docs', 'tests']), - install_requires=[ - "Django>=2.2.20,<3", - "django-guardian", - "django-registration-redux", - "django-templated-email", - "django-reversion", - "django-filter", - "django-nose", - "django-ordered-model", - # generic relation - "django-query-builder", - "celery", - # cors - "django-cors-headers", - "django-debug-toolbar", - # oauth2 support - "django-oauth-toolkit", - # "oauth2client", - "jsonpickle", - # jwt - "PyJWT", - # captcha - "recaptcha-client", - # API support - "djangorestframework", - "djangorestframework-csv", - "djangorestframework-gis", - "djangorestframework-guardian", - "djangorestframework-jsonapi", - "djangorestframework-jsonp", - "djangorestframework-xml", - # geojson - "geojson", - # tagging - "django-taggit", - # database - "psycopg2>2.7.1", - "pymongo", - # sms support - "dict2xml", - "lxml", - # pyxform - "pyxform", - # spss - "savreaderwriter", - # tests - "mock", - "httmock", - # JSON data type support, keeping it around for previous migration - "jsonfield<1.0", - # memcached support - "pylibmc", - "python-memcached", - # XML Instance API utility - "xmltodict", - # docs - "sphinx", - "Markdown", - # others - "unicodecsv", - "xlrd", - "xlwt", - "openpyxl", - "dpath", - "elaphe3", - "httplib2", - "modilabs-python-utils", - "numpy", - "Pillow", - "python-dateutil", - "pytz", - "requests", - "requests-mock", - "simplejson", - "uwsgi", - "flake8", - "raven", - "django-activity-stream", - "paho-mqtt", - "cryptography", - # Monitoring - "analytics-python", - "appoptics-metrics", - # Deprecation tagging - "deprecated", - # Redis cache - "django-redis", - ], - dependency_links=[ - 'https://github.com/onaio/python-digest/tarball/3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-digest/tarball/eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/django-multidb-router/tarball/9cf0a0c6c9f796e5bd14637fafeb5a1a5507ed37#egg=django-multidb-router', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/floip-py/tarball/3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=floip', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/python-json2xlsclient/tarball/62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient', # noqa pylint: disable=line-too-long - 'https://github.com/onaio/oauth2client/tarball/75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client', # noqa pylint: disable=line-too-long - ], - extras_require={ - ':python_version=="2.7"': [ - 'functools32>=3.2.3-2' - ] - } -) +if __name__ == "__main__": + setuptools.setup() From fed5113a32542fa04f812296c6cb43cae5c2fafb Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 01:18:55 +0300 Subject: [PATCH 009/234] Update basestring references --- onadata/apps/api/tasks.py | 25 +- .../tests/viewsets/test_attachment_viewset.py | 312 +- onadata/apps/api/viewsets/dataview_viewset.py | 232 +- .../apps/api/viewsets/user_profile_viewset.py | 238 +- .../management/commands/import_instances.py | 51 +- onadata/apps/logger/models/instance.py | 368 ++- onadata/apps/logger/models/widget.py | 117 +- .../apps/logger/tests/models/test_xform.py | 50 +- onadata/apps/main/models/meta_data.py | 247 +- .../apps/main/tests/test_form_enter_data.py | 27 +- onadata/apps/sms_support/tools.py | 358 +- onadata/libs/models/signals.py | 16 +- onadata/libs/serializers/chart_serializer.py | 23 +- onadata/libs/serializers/fields/json_field.py | 8 +- .../serializers/organization_serializer.py | 108 +- .../serializers/user_profile_serializer.py | 319 +- .../libs/tests/utils/test_export_builder.py | 2929 +++++++++-------- onadata/libs/utils/chart_tools.py | 290 +- onadata/libs/utils/common_tools.py | 48 +- onadata/libs/utils/csv_builder.py | 719 ++-- onadata/libs/utils/dict_tools.py | 41 +- onadata/libs/utils/qrcode.py | 35 +- onadata/libs/utils/string.py | 13 +- onadata/libs/utils/viewer_tools.py | 143 +- onadata/settings/common.py | 489 ++- 25 files changed, 3868 insertions(+), 3338 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index b8e1f4ff2f..6f965e7943 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -7,7 +7,6 @@ from django.core.files.storage import default_storage from django.contrib.auth.models import User from django.utils.datastructures import MultiValueDict -from past.builtins import basestring from onadata.apps.api import tools from onadata.libs.utils.email import send_generic_email @@ -26,7 +25,7 @@ def recreate_tmp_file(name, path, mime_type): def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): try: files = MultiValueDict() - files[u'xls_file'] = default_storage.open(file_data.get('path')) + files["xls_file"] = default_storage.open(file_data.get("path")) owner = User.objects.get(id=owner_id) if owner_id == user_id: @@ -34,7 +33,7 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): else: user = User.objects.get(id=user_id) survey = tools.do_publish_xlsform(user, post_data, files, owner) - default_storage.delete(file_data.get('path')) + default_storage.delete(file_data.get("path")) if isinstance(survey, XForm): return {"pk": survey.pk} @@ -46,13 +45,13 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): self.retry(exc=exc, countdown=1) else: error_message = ( - u'Service temporarily unavailable, please try to ' - 'publish the form again' + "Service temporarily unavailable, please try to " + "publish the form again" ) else: error_message = str(sys.exc_info()[1]) - return {u'error': error_message} + return {"error": error_message} @app.task() @@ -66,23 +65,23 @@ def delete_xform_async(xform_id, user_id): @app.task() def delete_user_async(): """Delete inactive user accounts""" - users = User.objects.filter(active=False, - username__contains="deleted-at", - email__contains="deleted-at") + users = User.objects.filter( + active=False, username__contains="deleted-at", email__contains="deleted-at" + ) for user in users: user.delete() def get_async_status(job_uuid): - """ Gets progress status or result """ + """Gets progress status or result""" if not job_uuid: - return {u'error': u'Empty job uuid'} + return {"error": "Empty job uuid"} job = AsyncResult(job_uuid) result = job.result or job.state - if isinstance(result, basestring): - return {'JOB_STATUS': result} + if isinstance(result, str): + return {"JOB_STATUS": result} return result diff --git a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py index 2ef1815985..4026d6a500 100644 --- a/onadata/apps/api/tests/viewsets/test_attachment_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_attachment_viewset.py @@ -1,11 +1,9 @@ import os -from past.builtins import basestring from django.utils import timezone -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models.attachment import Attachment @@ -18,22 +16,15 @@ def attachment_url(attachment, suffix=None): path = get_attachment_url(attachment, suffix) - return u'http://testserver{}'.format(path) + return "http://testserver{}".format(path) class TestAttachmentViewSet(TestAbstractViewSet): - def setUp(self): super(TestAttachmentViewSet, self).setUp() - self.retrieve_view = AttachmentViewSet.as_view({ - 'get': 'retrieve' - }) - self.list_view = AttachmentViewSet.as_view({ - 'get': 'list' - }) - self.count_view = AttachmentViewSet.as_view({ - 'get': 'count' - }) + self.retrieve_view = AttachmentViewSet.as_view({"get": "retrieve"}) + self.list_view = AttachmentViewSet.as_view({"get": "list"}) + self.count_view = AttachmentViewSet.as_view({"get": "count"}) self._publish_xls_form_to_project() @@ -43,36 +34,36 @@ def test_retrieve_view(self): pk = self.attachment.pk data = { - 'url': 'http://testserver/api/v1/media/%s' % pk, - 'field_xpath': 'image1', - 'download_url': attachment_url(self.attachment), - 'small_download_url': attachment_url(self.attachment, 'small'), - 'medium_download_url': attachment_url(self.attachment, 'medium'), - 'id': pk, - 'xform': self.xform.pk, - 'instance': self.attachment.instance.pk, - 'mimetype': self.attachment.mimetype, - 'filename': self.attachment.media_file.name + "url": "http://testserver/api/v1/media/%s" % pk, + "field_xpath": "image1", + "download_url": attachment_url(self.attachment), + "small_download_url": attachment_url(self.attachment, "small"), + "medium_download_url": attachment_url(self.attachment, "medium"), + "id": pk, + "xform": self.xform.pk, + "instance": self.attachment.instance.pk, + "mimetype": self.attachment.mimetype, + "filename": self.attachment.media_file.name, } - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, dict)) self.assertEqual(response.data, data) # file download - filename = data['filename'] - ext = filename[filename.rindex('.') + 1:] - request = self.factory.get('/', **self.extra) + filename = data["filename"] + ext = filename[filename.rindex(".") + 1 :] + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk, format=ext) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, 'image/jpeg') + self.assertEqual(response.content_type, "image/jpeg") self.attachment.instance.xform.deleted_at = timezone.now() self.attachment.instance.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) @@ -83,41 +74,46 @@ def test_attachment_pagination(self): self._submit_transport_instance_w_attachment() self.assertEqual(self.response.status_code, 201) filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "image2", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='image/jpeg', - extension='JPG', + mimetype="image/jpeg", + extension="JPG", name=filename, - media_file=media_file) + media_file=media_file, + ) # not using pagination params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # valid page and page_size - request = self.factory.get( - '/', data={"page": 1, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 1, "page_size": 1}, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) # invalid page type - request = self.factory.get('/', data={"page": "invalid"}, **self.extra) + request = self.factory.get("/", data={"page": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size type - request = self.factory.get('/', data={"page_size": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) @@ -125,14 +121,13 @@ def test_attachment_pagination(self): # invalid page and page_size types request = self.factory.get( - '/', data={"page": "invalid", "page_size": "invalid"}, - **self.extra) + "/", data={"page": "invalid", "page_size": "invalid"}, **self.extra + ) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size - request = self.factory.get( - '/', data={"page": 4, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 4, "page_size": 1}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -143,16 +138,16 @@ def test_retrieve_and_list_views_with_anonymous_user(self): pk = self.attachment.pk xform_id = self.attachment.instance.xform.id - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 404) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 404) @@ -160,24 +155,24 @@ def test_retrieve_and_list_views_with_anonymous_user(self): xform.shared_data = True xform.save() - request = self.factory.get('/') + request = self.factory.get("/") response = self.retrieve_view(request, pk=pk) self.assertEqual(response.status_code, 200) - request = self.factory.get('/') + request = self.factory.get("/") response = self.list_view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', data={"xform": xform_id}) + request = self.factory.get("/", data={"xform": xform_id}) response = self.list_view(request) self.assertEqual(response.status_code, 200) def test_list_view(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) @@ -186,7 +181,7 @@ def test_list_view(self): self.attachment.instance.deleted_at = timezone.now() self.attachment.instance.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 0) @@ -194,16 +189,16 @@ def test_list_view(self): def test_data_list_with_xform_in_delete_async(self): self._submit_transport_instance_w_attachment() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), initial_count - 1) @@ -211,204 +206,215 @@ def test_data_list_with_xform_in_delete_async(self): def test_list_view_filter_by_xform(self): self._submit_transport_instance_w_attachment() - data = { - 'xform': self.xform.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"xform": self.xform.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['xform'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["xform"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['xform'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["xform"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_instance(self): self._submit_transport_instance_w_attachment() - data = { - 'instance': self.attachment.instance.pk - } - request = self.factory.get('/', data, **self.extra) + data = {"instance": self.attachment.instance.pk} + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) - data['instance'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["instance"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) - data['instance'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["instance"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_list_view_filter_by_attachment_type(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - media_file = django_file(path, 'video2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + media_file = django_file(path, "video2", "image/jpeg") # test geojson attachments - geojson_filename = 'sample.geojson' - geojson_path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', - self.surveys[0], geojson_filename) - geojson_media_file = django_file( - geojson_path, 'store_gps', 'image/jpeg') + geojson_filename = "sample.geojson" + geojson_path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + geojson_filename, + ) + geojson_media_file = django_file(geojson_path, "store_gps", "image/jpeg") Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='video/mp4', - extension='MP4', + mimetype="video/mp4", + extension="MP4", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='application/pdf', - extension='PDF', + mimetype="application/pdf", + extension="PDF", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='text/plain', - extension='TXT', + mimetype="text/plain", + extension="TXT", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='audio/mp3', - extension='MP3', + mimetype="audio/mp3", + extension="MP3", name=filename, - media_file=media_file) + media_file=media_file, + ) Attachment.objects.create( instance=self.xform.instances.first(), - mimetype='application/geo+json', - extension='GEOJSON', + mimetype="application/geo+json", + extension="GEOJSON", name=geojson_filename, - media_file=geojson_media_file) + media_file=geojson_media_file, + ) data = {} - request = self.factory.get('/', data, **self.extra) + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 6) # Apply image Filter - data['type'] = 'image' - request = self.factory.get('/', data, **self.extra) + data["type"] = "image" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'image/jpeg') + self.assertEqual(response.data[0]["mimetype"], "image/jpeg") # Apply audio filter - data['type'] = 'audio' - request = self.factory.get('/', data, **self.extra) + data["type"] = "audio" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'audio/mp3') + self.assertEqual(response.data[0]["mimetype"], "audio/mp3") # Apply video filter - data['type'] = 'video' - request = self.factory.get('/', data, **self.extra) + data["type"] = "video" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) - self.assertEqual(response.data[0]["mimetype"], 'video/mp4') + self.assertEqual(response.data[0]["mimetype"], "video/mp4") # Apply file filter - data['type'] = 'document' - request = self.factory.get('/', data, **self.extra) + data["type"] = "document" + request = self.factory.get("/", data, **self.extra) response = self.list_view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 3) - self.assertEqual(response.data[0]["mimetype"], 'application/pdf') - self.assertEqual(response.data[1]["mimetype"], 'text/plain') - self.assertEqual(response.data[2]["mimetype"], 'application/geo+json') + self.assertEqual(response.data[0]["mimetype"], "application/pdf") + self.assertEqual(response.data[1]["mimetype"], "text/plain") + self.assertEqual(response.data[2]["mimetype"], "application/geo+json") def test_direct_image_link(self): self._submit_transport_instance_w_attachment() - data = { - 'filename': self.attachment.media_file.name - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": self.attachment.media_file.name} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) - data['filename'] = 10000000 - request = self.factory.get('/', data, **self.extra) + data["filename"] = 10000000 + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) - data['filename'] = 'lol' - request = self.factory.get('/', data, **self.extra) + data["filename"] = "lol" + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.instance.pk) self.assertEqual(response.status_code, 404) def test_direct_image_link_uppercase(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" - path = os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], filename) - self.attachment.media_file = django_file(path, 'image2', 'image/jpeg') + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + filename, + ) + self.attachment.media_file = django_file(path, "image2", "image/jpeg") self.attachment.name = filename self.attachment.save() filename = self.attachment.media_file.name file_base, file_extension = os.path.splitext(filename) - data = { - 'filename': file_base + file_extension.upper() - } - request = self.factory.get('/', data, **self.extra) + data = {"filename": file_base + file_extension.upper()} + request = self.factory.get("/", data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertTrue(isinstance(response.data, basestring)) + self.assertTrue(isinstance(response.data, str)) self.assertEqual(response.data, attachment_url(self.attachment)) def test_total_count(self): self._submit_transport_instance_w_attachment() xform_id = self.attachment.instance.xform.id - request = self.factory.get( - '/count', data={"xform": xform_id}, **self.extra) + request = self.factory.get("/count", data={"xform": xform_id}, **self.extra) response = self.count_view(request) - self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data["count"], 1) def test_returned_attachments_is_based_on_form_permissions(self): # Create a form and make submissions with attachments self._submit_transport_instance_w_attachment() formid = self.xform.pk - request = self.factory.get( - '/', data={"xform": formid}, **self.extra) + request = self.factory.get("/", data={"xform": formid}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -419,13 +425,11 @@ def test_returned_attachments_is_based_on_form_permissions(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) ShareXForm(self.xform, user_dave.username, EditorRole.name) - auth_extra = { - 'HTTP_AUTHORIZATION': f'Token {user_dave.auth_token.key}' - } + auth_extra = {"HTTP_AUTHORIZATION": f"Token {user_dave.auth_token.key}"} # Dave user should not be able to view attachments for # submissions which they did not submit - request = self.factory.get('/', data={"xform": formid}, **auth_extra) + request = self.factory.get("/", data={"xform": formid}, **auth_extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) # Ensure no submissions are returned for the User diff --git a/onadata/apps/api/viewsets/dataview_viewset.py b/onadata/apps/api/viewsets/dataview_viewset.py index 28c6cc7a69..9eb5aa426a 100644 --- a/onadata/apps/api/viewsets/dataview_viewset.py +++ b/onadata/apps/api/viewsets/dataview_viewset.py @@ -1,5 +1,3 @@ -from past.builtins import basestring - from django.db.models.signals import post_delete, post_save from django.http import Http404, HttpResponseBadRequest @@ -16,8 +14,7 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.data_view import DataView from onadata.apps.viewer.models.export import Export -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.renderers import renderers @@ -25,16 +22,22 @@ from onadata.libs.serializers.dataview_serializer import DataViewSerializer from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils import common_tags -from onadata.libs.utils.api_export_tools import (custom_response_handler, - export_async_export_response, - include_hxl_row, - process_async_export, - response_for_format) -from onadata.libs.utils.cache_tools import (PROJECT_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.chart_tools import (get_chart_data_for_field, - get_field_from_field_name) +from onadata.libs.utils.api_export_tools import ( + custom_response_handler, + export_async_export_response, + include_hxl_row, + process_async_export, + response_for_format, +) +from onadata.libs.utils.cache_tools import ( + PROJECT_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.chart_tools import ( + get_chart_data_for_field, + get_field_from_field_name, +) from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl @@ -42,12 +45,12 @@ def get_form_field_chart_url(url, field): - return u'%s?field_name=%s' % (url, field) + return "%s?field_name=%s" % (url, field) -class DataViewViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - ModelViewSet): +class DataViewViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): """ A simple ViewSet for viewing and editing DataViews. """ @@ -55,7 +58,7 @@ class DataViewViewSet(AuthenticateHeaderMixin, queryset = DataView.objects.select_related() serializer_class = DataViewSerializer permission_classes = [DataViewViewsetPermissions] - lookup_field = 'pk' + lookup_field = "pk" renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ renderers.XLSRenderer, renderers.XLSXRenderer, @@ -66,49 +69,54 @@ class DataViewViewSet(AuthenticateHeaderMixin, ] def get_serializer_class(self): - if self.action == 'data': + if self.action == "data": serializer_class = JsonDataSerializer else: serializer_class = self.serializer_class return serializer_class - @action(methods=['GET'], detail=True) - def data(self, request, format='json', **kwargs): + @action(methods=["GET"], detail=True) + def data(self, request, format="json", **kwargs): """Retrieve the data from the xform using this dataview""" start = request.GET.get("start") limit = request.GET.get("limit") count = request.GET.get("count") sort = request.GET.get("sort") query = request.GET.get("query") - export_type = self.kwargs.get('format', request.GET.get("format")) + export_type = self.kwargs.get("format", request.GET.get("format")) self.object = self.get_object() - if export_type is None or export_type in ['json', 'debug']: - data = DataView.query_data(self.object, start, limit, - str_to_bool(count), sort=sort, - filter_query=query) - if 'error' in data: - raise ParseError(data.get('error')) + if export_type is None or export_type in ["json", "debug"]: + data = DataView.query_data( + self.object, + start, + limit, + str_to_bool(count), + sort=sort, + filter_query=query, + ) + if "error" in data: + raise ParseError(data.get("error")) serializer = self.get_serializer(data, many=True) return Response(serializer.data) else: - return custom_response_handler(request, self.object.xform, query, - export_type, - dataview=self.object) + return custom_response_handler( + request, self.object.xform, query, export_type, dataview=self.object + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): params = request.query_params - job_uuid = params.get('job_uuid') - export_type = params.get('format') - include_hxl = params.get('include_hxl', False) - include_labels = params.get('include_labels', False) - include_labels_only = params.get('include_labels_only', False) - force_xlsx = params.get('force_xlsx', False) + job_uuid = params.get("job_uuid") + export_type = params.get("format") + include_hxl = params.get("include_hxl", False) + include_labels = params.get("include_labels", False) + include_labels_only = params.get("include_labels_only", False) + force_xlsx = params.get("force_xlsx", False) query = params.get("query") dataview = self.get_object() xform = dataview.xform @@ -125,100 +133,99 @@ def export_async(self, request, *args, **kwargs): if force_xlsx is not None: force_xlsx = str_to_bool(force_xlsx) - remove_group_name = params.get('remove_group_name', False) - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + remove_group_name = params.get("remove_group_name", False) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl and include_hxl: - include_hxl = include_hxl_row( - dataview.columns, list(columns_with_hxl) - ) + include_hxl = include_hxl_row(dataview.columns, list(columns_with_hxl)) options = { - 'remove_group_name': remove_group_name, - 'dataview_pk': dataview.pk, - 'include_hxl': include_hxl, - 'include_labels': include_labels, - 'include_labels_only': include_labels_only, - 'force_xlsx': force_xlsx, + "remove_group_name": remove_group_name, + "dataview_pk": dataview.pk, + "include_hxl": include_hxl, + "include_labels": include_labels, + "include_labels_only": include_labels_only, + "force_xlsx": force_xlsx, } if query: - options.update({'query': query}) + options.update({"query": query}) if job_uuid: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": export_id = job.result export = Export.objects.get(id=export_id) resp = export_async_export_response(request, export) else: - resp = { - 'job_status': job.state - } + resp = {"job_status": job.state} else: - resp = process_async_export(request, xform, export_type, - options=options) + resp = process_async_export(request, xform, export_type, options=options) - return Response(data=resp, - status=status.HTTP_202_ACCEPTED, - content_type="application/json") + return Response( + data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" + ) - @action(methods=['GET'], detail=True) - def form(self, request, format='json', **kwargs): + @action(methods=["GET"], detail=True) + def form(self, request, format="json", **kwargs): dataview = self.get_object() xform = dataview.xform - if format not in ['json', 'xml', 'xls']: - return HttpResponseBadRequest('400 BAD REQUEST', - content_type='application/json', - status=400) + if format not in ["json", "xml", "xls"]: + return HttpResponseBadRequest( + "400 BAD REQUEST", content_type="application/json", status=400 + ) filename = xform.id_string + "." + format response = response_for_format(xform, format=format) - response['Content-Disposition'] = 'attachment; filename=' + filename + response["Content-Disposition"] = "attachment; filename=" + filename return response - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def form_details(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform - serializer = XFormSerializer(xform, context={'request': request}) + serializer = XFormSerializer(xform, context={"request": request}) - return Response(data=serializer.data, - content_type="application/json") + return Response(data=serializer.data, content_type="application/json") - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def charts(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform serializer = self.get_serializer(dataview) - field_name = request.query_params.get('field_name') - field_xpath = request.query_params.get('field_xpath') - fmt = kwargs.get('format', request.accepted_renderer.format) - group_by = request.query_params.get('group_by') + field_name = request.query_params.get("field_name") + field_xpath = request.query_params.get("field_xpath") + fmt = kwargs.get("format", request.accepted_renderer.format) + group_by = request.query_params.get("group_by") if field_name: field = get_field_from_field_name(field_name, xform) - field_xpath = field_name if isinstance(field, basestring) \ - else field.get_abbreviated_xpath() + field_xpath = ( + field_name if isinstance(field, str) else field.get_abbreviated_xpath() + ) - if field_xpath and field_xpath not in dataview.columns and \ - field_xpath not in [common_tags.SUBMISSION_TIME, - common_tags.SUBMITTED_BY, - common_tags.DURATION]: - raise Http404( - "Field %s does not not exist on the dataview" % field_name) + if ( + field_xpath + and field_xpath not in dataview.columns + and field_xpath + not in [ + common_tags.SUBMISSION_TIME, + common_tags.SUBMITTED_BY, + common_tags.DURATION, + ] + ): + raise Http404("Field %s does not not exist on the dataview" % field_name) if field_name or field_xpath: data = get_chart_data_for_field( - field_name, xform, fmt, group_by, field_xpath, - data_view=dataview + field_name, xform, fmt, group_by, field_xpath, data_view=dataview ) - return Response(data, template_name='chart_detail.html') + return Response(data, template_name="chart_detail.html") - if fmt != 'json' and field_name is None: + if fmt != "json" and field_name is None: raise ParseError("Not supported") data = serializer.data @@ -226,14 +233,18 @@ def charts(self, request, *args, **kwargs): for field in xform.survey_elements: field_xpath = field.get_abbreviated_xpath() if field_xpath in dataview.columns: - url = reverse('dataviews-charts', kwargs={'pk': dataview.pk}, - request=request, format=fmt) + url = reverse( + "dataviews-charts", + kwargs={"pk": dataview.pk}, + request=request, + format=fmt, + ) field_url = get_form_field_chart_url(url, field.name) data["fields"][field.name] = field_url return Response(data) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def xls_export(self, request, *args, **kwargs): dataview = self.get_object() xform = dataview.xform @@ -241,39 +252,38 @@ def xls_export(self, request, *args, **kwargs): token = None export_type = "xls" query = request.query_params.get("query", {}) - meta = request.GET.get('meta') - return custom_response_handler(request, - xform, - query, - export_type, - token, - meta, - dataview) + meta = request.GET.get("meta") + + return custom_response_handler( + request, xform, query, export_type, token, meta, dataview + ) def destroy(self, request, *args, **kwargs): dataview = self.get_object() user = request.user dataview.soft_delete(user) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, dataview.project.pk)) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, dataview.project.pk)) return Response(status=status.HTTP_204_NO_CONTENT) -def dataview_post_save_callback(sender, instance=None, created=False, - **kwargs): - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) +def dataview_post_save_callback(sender, instance=None, created=False, **kwargs): + safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) def dataview_post_delete_callback(sender, instance, **kwargs): if instance.project: - safe_delete('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - instance.project.pk)) + safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) -post_save.connect(dataview_post_save_callback, - sender=DataView, - dispatch_uid='dataview_post_save_callback') +post_save.connect( + dataview_post_save_callback, + sender=DataView, + dispatch_uid="dataview_post_save_callback", +) -post_delete.connect(dataview_post_delete_callback, - sender=DataView, - dispatch_uid='dataview_post_delete_callback') +post_delete.connect( + dataview_post_delete_callback, + sender=DataView, + dispatch_uid="dataview_post_delete_callback", +) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 3aa8b18e12..16db8e1c42 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -7,7 +7,6 @@ import json from future.moves.urllib.parse import urlencode -from past.builtins import basestring # pylint: disable=redefined-builtin from django.conf import settings from django.core.cache import cache @@ -33,28 +32,27 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.libs.utils.email import (get_verification_email_data, - get_verification_url) -from onadata.libs.utils.cache_tools import (safe_delete, - CHANGE_PASSWORD_ATTEMPTS, - LOCKOUT_CHANGE_PASSWORD_USER, - USER_PROFILE_PREFIX) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url +from onadata.libs.utils.cache_tools import ( + safe_delete, + CHANGE_PASSWORD_ATTEMPTS, + LOCKOUT_CHANGE_PASSWORD_USER, + USER_PROFILE_PREFIX, +) from onadata.libs import filters from onadata.libs.utils.user_auth import invalidate_and_regen_tokens -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin -from onadata.libs.serializers.monthly_submissions_serializer import \ - MonthlySubmissionsSerializer -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileSerializer +from onadata.libs.serializers.monthly_submissions_serializer import ( + MonthlySubmissionsSerializer, +) +from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name -LOCKOUT_TIME = getattr(settings, 'LOCKOUT_TIME', 1800) -MAX_CHANGE_PASSWORD_ATTEMPTS = getattr( - settings, 'MAX_CHANGE_PASSWORD_ATTEMPTS', 10) +LOCKOUT_TIME = getattr(settings, "LOCKOUT_TIME", 1800) +MAX_CHANGE_PASSWORD_ATTEMPTS = getattr(settings, "MAX_CHANGE_PASSWORD_ATTEMPTS", 10) def replace_key_value(lookup, new_value, expected_dict): @@ -101,31 +99,30 @@ def serializer_from_settings(): def set_is_email_verified(profile, is_email_verified): - profile.metadata.update({'is_email_verified': is_email_verified}) + profile.metadata.update({"is_email_verified": is_email_verified}) profile.save() def check_user_lockout(request): username = request.user.username - lockout = cache.get('{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username)) + lockout = cache.get("{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username)) response_obj = { - 'error': 'Too many password reset attempts, Try again in {} minutes'} + "error": "Too many password reset attempts, Try again in {} minutes" + } if lockout: - time_locked_out = \ - datetime.datetime.now() - datetime.datetime.strptime( - lockout, '%Y-%m-%dT%H:%M:%S') - remaining_time = round( - (LOCKOUT_TIME - - time_locked_out.seconds) / 60) - response = response_obj['error'].format(remaining_time) + time_locked_out = datetime.datetime.now() - datetime.datetime.strptime( + lockout, "%Y-%m-%dT%H:%M:%S" + ) + remaining_time = round((LOCKOUT_TIME - time_locked_out.seconds) / 60) + response = response_obj["error"].format(remaining_time) return response def change_password_attempts(request): """Track number of login attempts made by user within a specified amount - of time""" + of time""" username = request.user.username - password_attempts = '{}{}'.format(CHANGE_PASSWORD_ATTEMPTS, username) + password_attempts = "{}{}".format(CHANGE_PASSWORD_ATTEMPTS, username) attempts = cache.get(password_attempts) if attempts: @@ -133,9 +130,10 @@ def change_password_attempts(request): attempts = cache.get(password_attempts) if attempts >= MAX_CHANGE_PASSWORD_ATTEMPTS: cache.set( - '{}{}'.format(LOCKOUT_CHANGE_PASSWORD_USER, username), - datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - LOCKOUT_TIME) + "{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username), + datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + LOCKOUT_TIME, + ) if check_user_lockout(request): return check_user_lockout(request) @@ -147,29 +145,32 @@ def change_password_attempts(request): class UserProfileViewSet( - AuthenticateHeaderMixin, # pylint: disable=R0901 - CacheControlMixin, - ETagsMixin, - ObjectLookupMixin, - BaseViewset, - ModelViewSet): + AuthenticateHeaderMixin, # pylint: disable=R0901 + CacheControlMixin, + ETagsMixin, + ObjectLookupMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create/Register users. """ - queryset = UserProfile.objects.select_related().filter( - user__is_active=True).exclude( - user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + + queryset = ( + UserProfile.objects.select_related() + .filter(user__is_active=True) + .exclude(user__username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + ) serializer_class = serializer_from_settings() - lookup_field = 'user' + lookup_field = "user" permission_classes = [UserProfilePermissions] filter_backends = (filters.UserProfileFilter, OrderingFilter) - ordering = ('user__username', ) + ordering = ("user__username",) def get_object(self, queryset=None): """Lookup user profile by pk or username""" if self.kwargs.get(self.lookup_field, None) is None: - raise ParseError( - 'Expected URL keyword argument `%s`.' % self.lookup_field) + raise ParseError("Expected URL keyword argument `%s`." % self.lookup_field) if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -180,7 +181,7 @@ def get_object(self, queryset=None): if self.lookup_field in serializer.get_fields(): k = serializer.get_fields()[self.lookup_field] if isinstance(k, serializers.HyperlinkedRelatedField): - lookup_field = '%s__%s' % (self.lookup_field, k.lookup_field) + lookup_field = "%s__%s" % (self.lookup_field, k.lookup_field) lookup = self.kwargs[self.lookup_field] filter_kwargs = {lookup_field: lookup} @@ -188,9 +189,9 @@ def get_object(self, queryset=None): try: user_pk = int(lookup) except (TypeError, ValueError): - filter_kwargs = {'%s__iexact' % lookup_field: lookup} + filter_kwargs = {"%s__iexact" % lookup_field: lookup} else: - filter_kwargs = {'user__pk': user_pk} + filter_kwargs = {"user__pk": user_pk} obj = get_object_or_404(queryset, **filter_kwargs) @@ -200,84 +201,72 @@ def get_object(self, queryset=None): return obj def update(self, request, *args, **kwargs): - """ Update user in cache and db""" - username = kwargs.get('user') - response = super(UserProfileViewSet, self)\ - .update(request, *args, **kwargs) - cache.set(f'{USER_PROFILE_PREFIX}{username}', response.data) + """Update user in cache and db""" + username = kwargs.get("user") + response = super(UserProfileViewSet, self).update(request, *args, **kwargs) + cache.set(f"{USER_PROFILE_PREFIX}{username}", response.data) return response def retrieve(self, request, *args, **kwargs): - """ Get user profile from cache or db """ - username = kwargs.get('user') - cached_user = cache.get(f'{USER_PROFILE_PREFIX}{username}') + """Get user profile from cache or db""" + username = kwargs.get("user") + cached_user = cache.get(f"{USER_PROFILE_PREFIX}{username}") if cached_user: return Response(cached_user) - response = super(UserProfileViewSet, self)\ - .retrieve(request, *args, **kwargs) + response = super(UserProfileViewSet, self).retrieve(request, *args, **kwargs) return response def create(self, request, *args, **kwargs): - """ Create and cache user profile """ - response = super(UserProfileViewSet, self)\ - .create(request, *args, **kwargs) + """Create and cache user profile""" + response = super(UserProfileViewSet, self).create(request, *args, **kwargs) profile = response.data - user_name = profile.get('username') - cache.set(f'{USER_PROFILE_PREFIX}{user_name}', profile) + user_name = profile.get("username") + cache.set(f"{USER_PROFILE_PREFIX}{user_name}", profile) return response - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def change_password(self, request, *args, **kwargs): # noqa """ Change user's password. """ # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") user_profile = self.get_object() - current_password = request.data.get('current_password', None) - new_password = request.data.get('new_password', None) + current_password = request.data.get("current_password", None) + new_password = request.data.get("new_password", None) lock_out = check_user_lockout(request) - response_obj = { - 'error': 'Invalid password. You have {} attempts left.'} + response_obj = {"error": "Invalid password. You have {} attempts left."} if new_password: if not lock_out: if user_profile.user.check_password(current_password): - data = { - 'username': user_profile.user.username - } + data = {"username": user_profile.user.username} metadata = user_profile.metadata or {} - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() user_profile.user.set_password(new_password) user_profile.metadata = metadata user_profile.user.save() user_profile.save() - data.update(invalidate_and_regen_tokens( - user=user_profile.user)) + data.update(invalidate_and_regen_tokens(user=user_profile.user)) - return Response( - status=status.HTTP_200_OK, data=data) + return Response(status=status.HTTP_200_OK, data=data) response = change_password_attempts(request) if isinstance(response, int): - limits_remaining = \ - MAX_CHANGE_PASSWORD_ATTEMPTS - response - response = response_obj['error'].format( - limits_remaining) - return Response(data=response, - status=status.HTTP_400_BAD_REQUEST) + limits_remaining = MAX_CHANGE_PASSWORD_ATTEMPTS - response + response = response_obj["error"].format(limits_remaining) + return Response(data=response, status=status.HTTP_400_BAD_REQUEST) return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) def partial_update(self, request, *args, **kwargs): profile = self.get_object() metadata = profile.metadata or {} - if request.data.get('overwrite') == 'false': - if isinstance(request.data.get('metadata'), basestring): - metadata_items = json.loads( - request.data.get('metadata')).items() + if request.data.get("overwrite") == "false": + if isinstance(request.data.get("metadata"), str): + metadata_items = json.loads(request.data.get("metadata")).items() else: - metadata_items = request.data.get('metadata').items() + metadata_items = request.data.get("metadata").items() for key, value in metadata_items: if check_if_key_exists(key, metadata): @@ -289,26 +278,24 @@ def partial_update(self, request, *args, **kwargs): profile.save() return Response(data=profile.metadata, status=status.HTTP_200_OK) - return super(UserProfileViewSet, self).partial_update( - request, *args, **kwargs) + return super(UserProfileViewSet, self).partial_update(request, *args, **kwargs) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def monthly_submissions(self, request, *args, **kwargs): - """ Get the total number of submissions for a user """ + """Get the total number of submissions for a user""" # clear cache - safe_delete(f'{USER_PROFILE_PREFIX}{request.user.username}') + safe_delete(f"{USER_PROFILE_PREFIX}{request.user.username}") profile = self.get_object() - month_param = self.request.query_params.get('month', None) - year_param = self.request.query_params.get('year', None) + month_param = self.request.query_params.get("month", None) + year_param = self.request.query_params.get("year", None) # check if parameters are valid if month_param: - if not month_param.isdigit() or \ - int(month_param) not in range(1, 13): - raise ValidationError(u'Invalid month provided as parameter') + if not month_param.isdigit() or int(month_param) not in range(1, 13): + raise ValidationError("Invalid month provided as parameter") if year_param: if not year_param.isdigit() or len(year_param) != 4: - raise ValidationError(u'Invalid year provided as parameter') + raise ValidationError("Invalid year provided as parameter") # Use query parameter values for month and year # if none, use the current month and year @@ -316,12 +303,16 @@ def monthly_submissions(self, request, *args, **kwargs): month = month_param if month_param else now.month year = year_param if year_param else now.year - instance_count = Instance.objects.filter( - xform__user=profile.user, - xform__deleted_at__isnull=True, - date_created__year=year, - date_created__month=month).values('xform__shared').annotate( - num_instances=Count('id')) + instance_count = ( + Instance.objects.filter( + xform__user=profile.user, + xform__deleted_at__isnull=True, + date_created__year=year, + date_created__month=month, + ) + .values("xform__shared") + .annotate(num_instances=Count("id")) + ) serializer = MonthlySubmissionsSerializer(instance_count, many=True) return Response(serializer.data[0]) @@ -333,21 +324,21 @@ def verify_email(self, request, *args, **kwargs): if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - redirect_url = request.query_params.get('redirect_url') - verification_key = request.query_params.get('verification_key') + redirect_url = request.query_params.get("redirect_url") + verification_key = request.query_params.get("verification_key") response_message = _("Missing or invalid verification key") if verification_key: rp = None try: rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + "user", "user__profile" + ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: with use_master: try: rp = RegistrationProfile.objects.select_related( - 'user', 'user__profile').get( - activation_key=verification_key) + "user", "user__profile" + ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: pass @@ -358,17 +349,13 @@ def verify_email(self, request, *args, **kwargs): username = rp.user.username set_is_email_verified(rp.user.profile, True) # Clear profiles cache - safe_delete(f'{USER_PROFILE_PREFIX}{username}') + safe_delete(f"{USER_PROFILE_PREFIX}{username}") - response_data = { - 'username': username, - 'is_email_verified': True - } + response_data = {"username": username, "is_email_verified": True} if redirect_url: query_params_string = urlencode(response_data) - redirect_url = '{}?{}'.format(redirect_url, - query_params_string) + redirect_url = "{}?{}".format(redirect_url, query_params_string) return HttpResponseRedirect(redirect_url) @@ -376,14 +363,14 @@ def verify_email(self, request, *args, **kwargs): return HttpResponseBadRequest(response_message) - @action(methods=['POST'], detail=False) + @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) - username = request.data.get('username') - redirect_url = request.data.get('redirect_url') + username = request.data.get("username") + redirect_url = request.data.get("redirect_url") response_message = _("Verification email has NOT been sent") if username: @@ -396,14 +383,17 @@ def send_verification_email(self, request, *args, **kwargs): verification_key = rp.activation_key if verification_key == verified_key_text: - verification_key = (rp.user.registrationprofile. - create_new_activation_key()) + verification_key = ( + rp.user.registrationprofile.create_new_activation_key() + ) verification_url = get_verification_url( - redirect_url, request, verification_key) + redirect_url, request, verification_key + ) email_data = get_verification_email_data( - rp.user.email, rp.user.username, verification_url, request) + rp.user.email, rp.user.username, verification_url, request + ) send_verification_email.delay(**email_data) response_message = _("Verification email has been sent") diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index eab0d33195..ab12285dd6 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -2,29 +2,34 @@ # vim: ai ts=4 sts=4 et sw=5 coding=utf-8 import os -from past.builtins import basestring from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from onadata.apps.logger.import_tools import (import_instances_from_path, - import_instances_from_zip) +from onadata.apps.logger.import_tools import ( + import_instances_from_path, + import_instances_from_zip, +) class Command(BaseCommand): - args = 'username path' - help = ugettext_lazy("Import a zip file, a directory containing zip files " - "or a directory of ODK instances") + args = "username path" + help = ugettext_lazy( + "Import a zip file, a directory containing zip files " + "or a directory of ODK instances" + ) def _log_import(self, results): total_count, success_count, errors = results - self.stdout.write(_( - "Total: %(total)d, Imported: %(imported)d, Errors: " - "%(errors)s\n------------------------------\n") % { - 'total': total_count, 'imported': success_count, - 'errors': errors}) + self.stdout.write( + _( + "Total: %(total)d, Imported: %(imported)d, Errors: " + "%(errors)s\n------------------------------\n" + ) + % {"total": total_count, "imported": success_count, "errors": errors} + ) def handle(self, *args, **kwargs): if len(args) < 2: @@ -32,18 +37,17 @@ def handle(self, *args, **kwargs): username = args[0] path = args[1] is_async = args[2] if len(args) > 2 else False - is_async = True if isinstance(is_async, basestring) and \ - is_async.lower() == 'true' else False + is_async = ( + True if isinstance(is_async, str) and is_async.lower() == "true" else False + ) try: user = User.objects.get(username=username) except User.DoesNotExist: - raise CommandError(_( - "The specified user '%s' does not exist.") % username) + raise CommandError(_("The specified user '%s' does not exist.") % username) # make sure path exists if not os.path.exists(path): - raise CommandError(_( - "The specified path '%s' does not exist.") % path) + raise CommandError(_("The specified path '%s' does not exist.") % path) for dir, subdirs, files in os.walk(path): # check if the dir has an odk directory @@ -51,15 +55,14 @@ def handle(self, *args, **kwargs): # dont walk further down this dir subdirs.remove("odk") self.stdout.write(_("Importing from dir %s..\n") % dir) - results = import_instances_from_path( - dir, user, is_async=is_async - ) + results = import_instances_from_path(dir, user, is_async=is_async) self._log_import(results) for file in files: filepath = os.path.join(path, file) - if os.path.isfile(filepath) and\ - os.path.splitext(filepath)[1].lower() == ".zip": - self.stdout.write(_( - "Importing from zip at %s..\n") % filepath) + if ( + os.path.isfile(filepath) + and os.path.splitext(filepath)[1].lower() == ".zip" + ): + self.stdout.write(_("Importing from zip at %s..\n") % filepath) results = import_instances_from_zip(filepath, user) self._log_import(results) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index ff85e527ce..fdd90df0b9 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -20,44 +20,74 @@ from django.utils import timezone from django.utils.translation import ugettext as _ from future.utils import python_2_unicode_compatible -from past.builtins import basestring # pylint: disable=W0622 from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview from onadata.apps.logger.models.survey_type import SurveyType from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH, XForm -from onadata.apps.logger.xform_instance_parser import (XFormInstanceParser, - clean_and_parse_xml, - get_uuid_from_xml) +from onadata.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + clean_and_parse_xml, + get_uuid_from_xml, +) from onadata.celery import app from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import ( - DATAVIEW_COUNT, IS_ORG, PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, - XFORM_COUNT, XFORM_DATA_VERSIONS, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) + DATAVIEW_COUNT, + IS_ORG, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + XFORM_DATA_VERSIONS, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, - DELETEDAT, DURATION, EDITED, END, GEOLOCATION, ID, LAST_EDITED, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, MONGO_STRFTIME, NOTES, - REVIEW_STATUS, START, STATUS, SUBMISSION_TIME, SUBMITTED_BY, - TAGS, TOTAL_MEDIA, UUID, VERSION, XFORM_ID, XFORM_ID_STRING, - REVIEW_COMMENT, REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + END, + GEOLOCATION, + ID, + LAST_EDITED, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MONGO_STRFTIME, + NOTES, + REVIEW_STATUS, + START, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID, + XFORM_ID_STRING, + REVIEW_COMMENT, + REVIEW_DATE, +) from onadata.libs.utils.dict_tools import get_values_matching_key from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.timing import calculate_duration -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) def get_attachment_url(attachment, suffix=None): - kwargs = {'pk': attachment.pk} - url = u'{}?filename={}'.format( - reverse('files-detail', kwargs=kwargs), - attachment.media_file.name + kwargs = {"pk": attachment.pk} + url = "{}?filename={}".format( + reverse("files-detail", kwargs=kwargs), attachment.media_file.name ) if suffix: - url += u'&suffix={}'.format(suffix) + url += "&suffix={}".format(suffix) return url @@ -66,15 +96,15 @@ def _get_attachments_from_instance(instance): attachments = [] for a in instance.attachments.filter(deleted_at__isnull=True): attachment = dict() - attachment['download_url'] = get_attachment_url(a) - attachment['small_download_url'] = get_attachment_url(a, 'small') - attachment['medium_download_url'] = get_attachment_url(a, 'medium') - attachment['mimetype'] = a.mimetype - attachment['filename'] = a.media_file.name - attachment['name'] = a.name - attachment['instance'] = a.instance.pk - attachment['xform'] = instance.xform.id - attachment['id'] = a.id + attachment["download_url"] = get_attachment_url(a) + attachment["small_download_url"] = get_attachment_url(a, "small") + attachment["medium_download_url"] = get_attachment_url(a, "medium") + attachment["mimetype"] = a.mimetype + attachment["filename"] = a.media_file.name + attachment["name"] = a.name + attachment["instance"] = a.instance.pk + attachment["xform"] = instance.xform.id + attachment["id"] = a.id attachments.append(attachment) return attachments @@ -89,15 +119,17 @@ def _get_tag_or_element_type_xpath(xform, tag): @python_2_unicode_compatible class FormInactiveError(Exception): """Exception class for inactive forms""" + def __str__(self): - return _(u'Form is inactive') + return _("Form is inactive") @python_2_unicode_compatible class FormIsMergedDatasetError(Exception): """Exception class for merged datasets""" + def __str__(self): - return _(u'Submissions are not allowed on merged datasets.') + return _("Submissions are not allowed on merged datasets.") def numeric_checker(string_value): @@ -113,6 +145,7 @@ def numeric_checker(string_value): return string_value + # need to establish id_string of the xform before we run get_dict since # we now rely on data dictionary to parse the xml @@ -120,15 +153,15 @@ def numeric_checker(string_value): def get_id_string_from_xml_str(xml_str): xml_obj = clean_and_parse_xml(xml_str) root_node = xml_obj.documentElement - id_string = root_node.getAttribute(u"id") + id_string = root_node.getAttribute("id") if len(id_string) == 0: # may be hidden in submission/data/id_string - elems = root_node.getElementsByTagName('data') + elems = root_node.getElementsByTagName("data") for data in elems: for child in data.childNodes: - id_string = data.childNodes[0].getAttribute('id') + id_string = data.childNodes[0].getAttribute("id") if len(id_string) > 0: break @@ -144,15 +177,17 @@ def submission_time(): def _update_submission_count_for_today( - form_id: int, incr: bool = True, date_created=None): + form_id: int, incr: bool = True, date_created=None +): # Track submissions made today current_timzone_name = timezone.get_current_timezone_name() current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - date_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}") - count_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}") + datetime(today.year, today.month, today.day) + ).isoformat() + date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}" + count_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}" if not cache.get(date_cache_key) == current_date: cache.set(date_cache_key, current_date, 86400) @@ -176,42 +211,45 @@ def _update_submission_count_for_today( def update_xform_submission_count(instance_id, created): if created: from multidb.pinning import use_master + with use_master: try: - instance = Instance.objects.select_related('xform').only( - 'xform__user_id', 'date_created').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform") + .only("xform__user_id", "date_created") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: # update xform.num_of_submissions cursor = connection.cursor() sql = ( - 'UPDATE logger_xform SET ' - 'num_of_submissions = num_of_submissions + 1, ' - 'last_submission_time = %s ' - 'WHERE id = %s' + "UPDATE logger_xform SET " + "num_of_submissions = num_of_submissions + 1, " + "last_submission_time = %s " + "WHERE id = %s" ) params = [instance.date_created, instance.xform_id] # update user profile.num_of_submissions cursor.execute(sql, params) sql = ( - 'UPDATE main_userprofile SET ' - 'num_of_submissions = num_of_submissions + 1 ' - 'WHERE user_id = %s' + "UPDATE main_userprofile SET " + "num_of_submissions = num_of_submissions + 1 " + "WHERE user_id = %s" ) cursor.execute(sql, [instance.xform.user_id]) # Track submissions made today _update_submission_count_for_today(instance.xform_id) - safe_delete('{}{}'.format( - XFORM_DATA_VERSIONS, instance.xform_id)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform_id)) - safe_delete('{}{}'.format(XFORM_COUNT, instance.xform_id)) + safe_delete("{}{}".format(XFORM_DATA_VERSIONS, instance.xform_id)) + safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform_id)) + safe_delete("{}{}".format(XFORM_COUNT, instance.xform_id)) # Clear project cache - from onadata.apps.logger.models.xform import \ - clear_project_cache + from onadata.apps.logger.models.xform import clear_project_cache + clear_project_cache(instance.xform.project_id) @@ -224,11 +262,10 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): xform.num_of_submissions -= 1 if xform.num_of_submissions < 0: xform.num_of_submissions = 0 - xform.save(update_fields=['num_of_submissions']) + xform.save(update_fields=["num_of_submissions"]) profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=xform.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=xform.user.profile.pk) except profile_qs.model.DoesNotExist: pass else: @@ -239,15 +276,16 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): # Track submissions made today _update_submission_count_for_today( - xform.id, incr=False, date_created=instance.date_created) + xform.id, incr=False, date_created=instance.date_created + ) for a in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: - safe_delete('{}{}'.format(a, xform.project.pk)) + safe_delete("{}{}".format(a, xform.project.pk)) - safe_delete('{}{}'.format(IS_ORG, xform.pk)) - safe_delete('{}{}'.format(XFORM_DATA_VERSIONS, xform.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, xform.pk)) - safe_delete('{}{}'.format(XFORM_COUNT, xform.pk)) + safe_delete("{}{}".format(IS_ORG, xform.pk)) + safe_delete("{}{}".format(XFORM_DATA_VERSIONS, xform.pk)) + safe_delete("{}{}".format(DATAVIEW_COUNT, xform.pk)) + safe_delete("{}{}".format(XFORM_COUNT, xform.pk)) if xform.instances.exclude(geom=None).count() < 1: xform.instances_with_geopoints = False @@ -264,7 +302,7 @@ def save_full_json(instance_id, created): pass else: instance.json = instance.get_full_dict() - instance.save(update_fields=['json']) + instance.save(update_fields=["json"]) @app.task @@ -272,16 +310,19 @@ def update_project_date_modified(instance_id, created): # update the date modified field of the project which will change # the etag value of the projects endpoint try: - instance = Instance.objects.select_related('xform__project').only( - 'xform__project__date_modified').get(pk=instance_id) + instance = ( + Instance.objects.select_related("xform__project") + .only("xform__project__date_modified") + .get(pk=instance_id) + ) except Instance.DoesNotExist: pass else: - instance.xform.project.save(update_fields=['date_modified']) + instance.xform.project.save(update_fields=["date_modified"]) def convert_to_serializable_date(date): - if hasattr(date, 'isoformat'): + if hasattr(date, "isoformat"): return date.isoformat() return date @@ -302,22 +343,20 @@ def numeric_converter(self, json_dict, numeric_fields=None): # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): - if isinstance(value, basestring) and key in numeric_fields: + if isinstance(value, str) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): - json_dict[key] = self.numeric_converter( - value, numeric_fields) + json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): - if isinstance(v, basestring) and key in numeric_fields: + if isinstance(v, str) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): - value[k] = self.numeric_converter( - v, numeric_fields) + value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _set_geom(self): @@ -352,22 +391,25 @@ def get_full_dict(self, load_existing=True): doc = self.get_dict() # pylint: disable=no-member if self.id: - doc.update({ - UUID: self.uuid, - ID: self.id, - BAMBOO_DATASET_ID: self.xform.bamboo_dataset, - ATTACHMENTS: _get_attachments_from_instance(self), - STATUS: self.status, - TAGS: list(self.tags.names()), - NOTES: self.get_notes(), - VERSION: self.version, - DURATION: self.get_duration(), - XFORM_ID_STRING: self._parser.get_xform_id_string(), - XFORM_ID: self.xform.pk, - GEOLOCATION: [self.point.y, self.point.x] if self.point - else [None, None], - SUBMITTED_BY: self.user.username if self.user else None - }) + doc.update( + { + UUID: self.uuid, + ID: self.id, + BAMBOO_DATASET_ID: self.xform.bamboo_dataset, + ATTACHMENTS: _get_attachments_from_instance(self), + STATUS: self.status, + TAGS: list(self.tags.names()), + NOTES: self.get_notes(), + VERSION: self.version, + DURATION: self.get_duration(), + XFORM_ID_STRING: self._parser.get_xform_id_string(), + XFORM_ID: self.xform.pk, + GEOLOCATION: [self.point.y, self.point.x] + if self.point + else [None, None], + SUBMITTED_BY: self.user.username if self.user else None, + } + ) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) @@ -380,8 +422,7 @@ def get_full_dict(self, load_existing=True): review = self.get_latest_review() if review: doc[REVIEW_STATUS] = review.status - doc[REVIEW_DATE] = review.date_created.strftime( - MONGO_STRFTIME) + doc[REVIEW_DATE] = review.date_created.strftime(MONGO_STRFTIME) if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() @@ -392,8 +433,7 @@ def get_full_dict(self, load_existing=True): if not self.date_modified: self.date_modified = self.date_created - doc[DATE_MODIFIED] = self.date_modified.strftime( - MONGO_STRFTIME) + doc[DATE_MODIFIED] = self.date_modified.strftime(MONGO_STRFTIME) doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) @@ -402,13 +442,13 @@ def get_full_dict(self, load_existing=True): doc[MEDIA_ALL_RECEIVED] = self.media_all_received edited = False - if hasattr(self, 'last_edited'): + if hasattr(self, "last_edited"): edited = self.last_edited is not None doc[EDITED] = edited - edited and doc.update({ - LAST_EDITED: convert_to_serializable_date(self.last_edited) - }) + edited and doc.update( + {LAST_EDITED: convert_to_serializable_date(self.last_edited)} + ) return doc def _set_parser(self): @@ -417,8 +457,9 @@ def _set_parser(self): self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): - self.survey_type, created = \ - SurveyType.objects.get_or_create(slug=self.get_root_node_name()) + self.survey_type, created = SurveyType.objects.get_or_create( + slug=self.get_root_node_name() + ) def _set_uuid(self): # pylint: disable=no-member, attribute-defined-outside-init @@ -437,24 +478,26 @@ def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() - instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ + instance_dict = ( + self._parser.get_flat_dict_with_attributes() + if flat else self._parser.to_dict() + ) return self.numeric_converter(instance_dict) def get_notes(self): # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] - @deprecated(version='2.5.3', - reason="Deprecated in favour of `get_latest_review`") + @deprecated(version="2.5.3", reason="Deprecated in favour of `get_latest_review`") def get_review_status_and_comment(self): """ Return a tuple of review status and comment. """ try: # pylint: disable=no-member - status = self.reviews.latest('date_modified').status - comment = self.reviews.latest('date_modified').get_note_text() + status = self.reviews.latest("date_modified").status + comment = self.reviews.latest("date_modified").get_note_text() return status, comment except SubmissionReview.DoesNotExist: return None @@ -482,7 +525,7 @@ def get_latest_review(self): Used in favour of `get_review_status_and_comment`. """ try: - return self.reviews.latest('date_modified') + return self.reviews.latest("date_modified") except SubmissionReview.DoesNotExist: return None @@ -495,12 +538,12 @@ class Instance(models.Model, InstanceBaseClass): json = JSONField(default=dict, null=False) xml = models.TextField() user = models.ForeignKey( - User, related_name='instances', null=True, on_delete=models.SET_NULL) + User, related_name="instances", null=True, on_delete=models.SET_NULL + ) xform = models.ForeignKey( - 'logger.XForm', null=False, related_name='instances', - on_delete=models.CASCADE) - survey_type = models.ForeignKey( - 'logger.SurveyType', on_delete=models.PROTECT) + "logger.XForm", null=False, related_name="instances", on_delete=models.CASCADE + ) + survey_type = models.ForeignKey("logger.SurveyType", on_delete=models.PROTECT) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) @@ -510,8 +553,9 @@ class Instance(models.Model, InstanceBaseClass): # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) - deleted_by = models.ForeignKey(User, related_name='deleted_instances', - null=True, on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, related_name="deleted_instances", null=True, on_delete=models.SET_NULL + ) # this will be edited when we need to create a new InstanceHistory object last_edited = models.DateTimeField(null=True, default=None) @@ -521,9 +565,8 @@ class Instance(models.Model, InstanceBaseClass): # we add the following additional statuses: # - submitted_via_web # - imported_via_csv - status = models.CharField(max_length=20, - default=u'submitted_via_web') - uuid = models.CharField(max_length=249, default=u'', db_index=True) + status = models.CharField(max_length=20, default="submitted_via_web") + uuid = models.CharField(max_length=249, default="", db_index=True) version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True) # store a geographic objects associated with this instance @@ -531,25 +574,23 @@ class Instance(models.Model, InstanceBaseClass): # Keep track of whether all media attachments have been received media_all_received = models.NullBooleanField( - _("Received All Media Attachemts"), - null=True, - default=True) - total_media = models.PositiveIntegerField(_("Total Media Attachments"), - null=True, - default=0) - media_count = models.PositiveIntegerField(_("Received Media Attachments"), - null=True, - default=0) - checksum = models.CharField(max_length=64, null=True, blank=True, - db_index=True) + _("Received All Media Attachemts"), null=True, default=True + ) + total_media = models.PositiveIntegerField( + _("Total Media Attachments"), null=True, default=0 + ) + media_count = models.PositiveIntegerField( + _("Received Media Attachments"), null=True, default=0 + ) + checksum = models.CharField(max_length=64, null=True, blank=True, db_index=True) # Keep track of submission reviews, only query reviews if true has_a_review = models.BooleanField(_("has_a_review"), default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = ('xform', 'uuid') + app_label = "logger" + unique_together = ("xform", "uuid") @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now(), user=None): @@ -582,21 +623,22 @@ def get_expected_media(self): """ Returns a list of expected media files from the submission data. """ - if not hasattr(self, '_expected_media'): + if not hasattr(self, "_expected_media"): # pylint: disable=no-member data = self.get_dict() media_list = [] - if 'encryptedXmlFile' in data and self.xform.encrypted: - media_list.append(data['encryptedXmlFile']) - if 'media' in data: + if "encryptedXmlFile" in data and self.xform.encrypted: + media_list.append(data["encryptedXmlFile"]) + if "media" in data: # pylint: disable=no-member - media_list.extend([i['media/file'] for i in data['media']]) + media_list.extend([i["media/file"] for i in data["media"]]) else: - media_xpaths = (self.xform.get_media_survey_xpaths() + - self.xform.get_osm_survey_xpaths()) + media_xpaths = ( + self.xform.get_media_survey_xpaths() + + self.xform.get_osm_survey_xpaths() + ) for media_xpath in media_xpaths: - media_list.extend( - get_values_matching_key(data, media_xpath)) + media_list.extend(get_values_matching_key(data, media_xpath)) # pylint: disable=attribute-defined-outside-init self._expected_media = list(set(media_list)) @@ -607,7 +649,7 @@ def num_of_media(self): """ Returns number of media attachments expected in the submission. """ - if not hasattr(self, '_num_of_media'): + if not hasattr(self, "_num_of_media"): # pylint: disable=attribute-defined-outside-init self._num_of_media = len(self.get_expected_media()) @@ -615,15 +657,18 @@ def num_of_media(self): @property def attachments_count(self): - return self.attachments.filter( - name__in=self.get_expected_media() - ).distinct('name').order_by('name').count() + return ( + self.attachments.filter(name__in=self.get_expected_media()) + .distinct("name") + .order_by("name") + .count() + ) def save(self, *args, **kwargs): - force = kwargs.get('force') + force = kwargs.get("force") if force: - del kwargs['force'] + del kwargs["force"] self._check_is_merged_dataset() self._check_active(force) @@ -650,19 +695,18 @@ def soft_delete_attachments(self, user=None): """ Soft deletes an attachment by adding a deleted_at timestamp. """ - queryset = self.attachments.filter( - ~Q(name__in=self.get_expected_media())) - kwargs = {'deleted_at': timezone.now()} + queryset = self.attachments.filter(~Q(name__in=self.get_expected_media())) + kwargs = {"deleted_at": timezone.now()} if user: - kwargs.update({'deleted_by': user}) + kwargs.update({"deleted_by": user}) queryset.update(**kwargs) def post_save_submission(sender, instance=None, created=False, **kwargs): if instance.deleted_at is not None: - _update_submission_count_for_today(instance.xform_id, - incr=False, - date_created=instance.date_created) + _update_submission_count_for_today( + instance.xform_id, incr=False, date_created=instance.date_created + ) if ASYNC_POST_SUBMISSION_PROCESSING_ENABLED: update_xform_submission_count.apply_async(args=[instance.pk, created]) @@ -675,25 +719,29 @@ def post_save_submission(sender, instance=None, created=False, **kwargs): update_project_date_modified(instance.pk, created) -post_save.connect(post_save_submission, sender=Instance, - dispatch_uid='post_save_submission') +post_save.connect( + post_save_submission, sender=Instance, dispatch_uid="post_save_submission" +) -post_delete.connect(update_xform_submission_count_delete, sender=Instance, - dispatch_uid='update_xform_submission_count_delete') +post_delete.connect( + update_xform_submission_count_delete, + sender=Instance, + dispatch_uid="update_xform_submission_count_delete", +) class InstanceHistory(models.Model, InstanceBaseClass): - class Meta: - app_label = 'logger' + app_label = "logger" xform_instance = models.ForeignKey( - Instance, related_name='submission_history', on_delete=models.CASCADE) + Instance, related_name="submission_history", on_delete=models.CASCADE + ) user = models.ForeignKey(User, null=True, on_delete=models.CASCADE) xml = models.TextField() # old instance id - uuid = models.CharField(max_length=249, default=u'') + uuid = models.CharField(max_length=249, default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -759,9 +807,7 @@ def media_all_received(self): def _set_parser(self): if not hasattr(self, "_parser"): - self._parser = XFormInstanceParser( - self.xml, self.xform_instance.xform - ) + self._parser = XFormInstanceParser(self.xml, self.xform_instance.xform) @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): diff --git a/onadata/apps/logger/models/widget.py b/onadata/apps/logger/models/widget.py index cfb48fb7ff..67b447ecc1 100644 --- a/onadata/apps/logger/models/widget.py +++ b/onadata/apps/logger/models/widget.py @@ -1,5 +1,4 @@ from builtins import str as text -from past.builtins import basestring from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType @@ -12,50 +11,46 @@ from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm -from onadata.libs.utils.chart_tools import (DATA_TYPE_MAP, - _flatten_multiple_dict_into_one, - _use_labels_from_group_by_name, - get_field_choices, - get_field_from_field_xpath, - get_field_label) -from onadata.libs.utils.common_tags import (NUMERIC_LIST, SELECT_ONE, - SUBMISSION_TIME) +from onadata.libs.utils.chart_tools import ( + DATA_TYPE_MAP, + _flatten_multiple_dict_into_one, + _use_labels_from_group_by_name, + get_field_choices, + get_field_from_field_xpath, + get_field_label, +) +from onadata.libs.utils.common_tags import NUMERIC_LIST, SELECT_ONE, SUBMISSION_TIME from onadata.libs.utils.common_tools import get_uuid class Widget(OrderedModel): - CHARTS = 'charts' + CHARTS = "charts" # Other widgets types to be added later - WIDGETS_TYPES = ((CHARTS, 'Charts'), ) + WIDGETS_TYPES = ((CHARTS, "Charts"),) # Will hold either XForm or DataView Model content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") - widget_type = models.CharField( - max_length=25, choices=WIDGETS_TYPES, default=CHARTS) + widget_type = models.CharField(max_length=25, choices=WIDGETS_TYPES, default=CHARTS) view_type = models.CharField(max_length=50) column = models.CharField(max_length=255) - group_by = models.CharField( - null=True, default=None, max_length=255, blank=True) - - title = models.CharField( - null=True, default=None, max_length=255, blank=True) - description = models.CharField( - null=True, default=None, max_length=255, blank=True) - aggregation = models.CharField( - null=True, default=None, max_length=255, blank=True) + group_by = models.CharField(null=True, default=None, max_length=255, blank=True) + + title = models.CharField(null=True, default=None, max_length=255, blank=True) + description = models.CharField(null=True, default=None, max_length=255, blank=True) + aggregation = models.CharField(null=True, default=None, max_length=255, blank=True) key = models.CharField(db_index=True, unique=True, max_length=32) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - order_with_respect_to = 'content_type' + order_with_respect_to = "content_type" metadata = JSONField(default=dict, blank=True) class Meta(OrderedModel.Meta): - app_label = 'logger' + app_label = "logger" def save(self, *args, **kwargs): @@ -77,61 +72,60 @@ def query_data(cls, widget): field = get_field_from_field_xpath(column, xform) - if isinstance(field, basestring) and field == SUBMISSION_TIME: - field_label = 'Submission Time' - field_xpath = '_submission_time' - field_type = 'datetime' - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') + if isinstance(field, str) and field == SUBMISSION_TIME: + field_label = "Submission Time" + field_xpath = "_submission_time" + field_type = "datetime" + data_type = DATA_TYPE_MAP.get(field_type, "categorized") else: field_type = field.type - data_type = DATA_TYPE_MAP.get(field.type, 'categorized') + data_type = DATA_TYPE_MAP.get(field.type, "categorized") field_xpath = field.get_abbreviated_xpath() field_label = get_field_label(field) columns = [ - SimpleField( - field="json->>'%s'" % text(column), - alias='{}'.format(column)), - CountField( - field="json->>'%s'" % text(column), - alias='count') + SimpleField(field="json->>'%s'" % text(column), alias="{}".format(column)), + CountField(field="json->>'%s'" % text(column), alias="count"), ] if group_by: if field_type in NUMERIC_LIST: column_field = SimpleField( - field="json->>'%s'" % text(column), - cast="float", - alias=column) + field="json->>'%s'" % text(column), cast="float", alias=column + ) else: column_field = SimpleField( - field="json->>'%s'" % text(column), alias=column) + field="json->>'%s'" % text(column), alias=column + ) # build inner query - inner_query_columns = \ - [column_field, - SimpleField(field="json->>'%s'" % text(group_by), - alias=group_by), - SimpleField(field="xform_id"), - SimpleField(field="deleted_at")] + inner_query_columns = [ + column_field, + SimpleField(field="json->>'%s'" % text(group_by), alias=group_by), + SimpleField(field="xform_id"), + SimpleField(field="deleted_at"), + ] inner_query = Query().from_table(Instance, inner_query_columns) # build group-by query if field_type in NUMERIC_LIST: columns = [ - SimpleField(field=group_by, alias='%s' % group_by), + SimpleField(field=group_by, alias="%s" % group_by), SumField(field=column, alias="sum"), - AvgField(field=column, alias="mean") + AvgField(field=column, alias="mean"), ] elif field_type == SELECT_ONE: columns = [ - SimpleField(field=column, alias='%s' % column), - SimpleField(field=group_by, alias='%s' % group_by), - CountField(field="*", alias='count') + SimpleField(field=column, alias="%s" % column), + SimpleField(field=group_by, alias="%s" % group_by), + CountField(field="*", alias="count"), ] - query = Query().from_table({'inner_query': inner_query}, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table({"inner_query": inner_query}, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) if field_type == SELECT_ONE: query.group_by(column).group_by(group_by) @@ -139,8 +133,11 @@ def query_data(cls, widget): query.group_by(group_by) else: - query = Query().from_table(Instance, columns).\ - where(xform_id=xform.pk, deleted_at=None) + query = ( + Query() + .from_table(Instance, columns) + .where(xform_id=xform.pk, deleted_at=None) + ) query.group_by("json->>'%s'" % text(column)) # run query @@ -148,14 +145,14 @@ def query_data(cls, widget): # flatten multiple dict if select one with group by if field_type == SELECT_ONE and group_by: - records = _flatten_multiple_dict_into_one(column, group_by, - records) + records = _flatten_multiple_dict_into_one(column, group_by, records) # use labels if group by if group_by: group_by_field = get_field_from_field_xpath(group_by, xform) choices = get_field_choices(group_by, xform) records = _use_labels_from_group_by_name( - group_by, group_by_field, data_type, records, choices=choices) + group_by, group_by_field, data_type, records, choices=choices + ) return { "field_type": field_type, @@ -163,5 +160,5 @@ def query_data(cls, widget): "field_xpath": field_xpath, "field_label": field_label, "grouped_by": group_by, - "data": records + "data": records, } diff --git a/onadata/apps/logger/tests/models/test_xform.py b/onadata/apps/logger/tests/models/test_xform.py index 856145fc2b..3564f4c100 100644 --- a/onadata/apps/logger/tests/models/test_xform.py +++ b/onadata/apps/logger/tests/models/test_xform.py @@ -5,11 +5,9 @@ import os from builtins import str as text -from past.builtins import basestring # pylint: disable=redefined-builtin from onadata.apps.logger.models import Instance, XForm -from onadata.apps.logger.models.xform import (DuplicateUUIDError, - check_xform_uuid) +from onadata.apps.logger.models.xform import DuplicateUUIDError, check_xform_uuid from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.xform_instance_parser import XLSFormError @@ -18,6 +16,7 @@ class TestXForm(TestBase): """ Test XForm model. """ + def test_submission_count(self): """ Test submission count does not include deleted submissions. @@ -43,14 +42,17 @@ def test_set_title_unicode_error(self): """ xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../..", "fixtures", "tutorial", "tutorial_arabic_labels.xlsx" + "../..", + "fixtures", + "tutorial", + "tutorial_arabic_labels.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - self.assertTrue(isinstance(self.xform.xml, basestring)) + self.assertTrue(isinstance(self.xform.xml, str)) # change title - self.xform.title = u'Random Title' + self.xform.title = "Random Title" self.assertNotIn(self.xform.title, self.xform.xml) @@ -65,7 +67,7 @@ def test_version_length(self): """Test Xform.version can store more than 12 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - xform.version = u'12345678901234567890' + xform.version = "12345678901234567890" xform.save() self.assertTrue(len(xform.version) > 12) @@ -89,7 +91,7 @@ def test_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() @@ -103,7 +105,7 @@ def test_soft_delete(self): # deleted-at suffix is present self.assertIn("-deleted-at-", xform.id_string) self.assertIn("-deleted-at-", xform.sms_id_string) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_get_survey_element(self): """ @@ -127,7 +129,7 @@ def test_get_survey_element(self): | | fruity | orange | Orange | | | fruity | mango | Mango | """ - kwargs = {'name': 'favs', 'title': 'Fruits', 'id_string': 'favs'} + kwargs = {"name": "favs", "title": "Fruits", "id_string": "favs"} survey = self.md_to_pyxform_survey(markdown_xlsform, kwargs) xform = XForm() xform._survey = survey # pylint: disable=W0212 @@ -136,7 +138,7 @@ def test_get_survey_element(self): self.assertIsNone(xform.get_survey_element("non_existent")) # get fruita element by name - fruita = xform.get_survey_element('fruita') + fruita = xform.get_survey_element("fruita") self.assertEqual(fruita.get_abbreviated_xpath(), "a/fruita") # get exact choices element from choice abbreviated xpath @@ -149,7 +151,7 @@ def test_get_survey_element(self): fruitb_o = xform.get_survey_element("b/fruitb/orange") self.assertEqual(fruitb_o.get_abbreviated_xpath(), "b/fruitb/orange") - self.assertEqual(xform.get_child_elements('NoneExistent'), []) + self.assertEqual(xform.get_child_elements("NoneExistent"), []) def test_check_xform_uuid(self): """ @@ -173,8 +175,10 @@ def test_id_string_max_length_on_soft_delete(self): """ self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string" + ) xform.id_string = new_string xform.sms_id_string = new_string @@ -187,15 +191,13 @@ def test_id_string_max_length_on_soft_delete(self): # '&' should raise an XLSFormError exception when being changed, for # deletions this should not raise any exception however - xform.title = 'Trial & Error' + xform.title = "Trial & Error" xform.soft_delete(self.user) xform.refresh_from_db() - d_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') - d_sms_id_string = new_string + xform.deleted_at.strftime( - '-deleted-at-%s') + d_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") + d_sms_id_string = new_string + xform.deleted_at.strftime("-deleted-at-%s") # deleted_at is not None self.assertIsNotNone(xform.deleted_at) @@ -209,15 +211,17 @@ def test_id_string_max_length_on_soft_delete(self): self.assertIn(xform.sms_id_string, d_sms_id_string) self.assertEqual(xform.id_string, d_id_string[:100]) self.assertEqual(xform.sms_id_string, d_sms_id_string[:100]) - self.assertEqual(xform.deleted_by.username, 'bob') + self.assertEqual(xform.deleted_by.username, "bob") def test_id_string_length(self): """Test Xform.id_string cannot store more than 100 chars""" self._publish_transportation_form_and_submit_instance() xform = XForm.objects.get(pk=self.xform.id) - new_string = "transportation_twenty_fifth_july_two_thousand_and_" \ - "eleven_test_for_long_sms_id_string_and_id_string_" \ - "before_save" + new_string = ( + "transportation_twenty_fifth_july_two_thousand_and_" + "eleven_test_for_long_sms_id_string_and_id_string_" + "before_save" + ) xform.id_string = new_string xform.sms_id_string = new_string diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index ecb3f599d1..aab4dbf687 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -17,11 +17,14 @@ from django.core.validators import URLValidator from django.db import IntegrityError, models from django.db.models.signals import post_delete, post_save -from past.builtins import basestring from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE, safe_delete -from onadata.libs.utils.common_tags import (GOOGLE_SHEET_DATA_TYPE, TEXTIT, - XFORM_META_PERMS, TEXTIT_DETAILS) +from onadata.libs.utils.common_tags import ( + GOOGLE_SHEET_DATA_TYPE, + TEXTIT, + XFORM_META_PERMS, + TEXTIT_DETAILS, +) CHUNK_SIZE = 1024 INSTANCE_MODEL_NAME = "instance" @@ -46,16 +49,18 @@ def is_valid_url(uri): def upload_to(instance, filename): username = None - if instance.content_object.user is None and \ - instance.content_type.model == INSTANCE_MODEL_NAME: + if ( + instance.content_object.user is None + and instance.content_type.model == INSTANCE_MODEL_NAME + ): username = instance.content_object.xform.user.username else: username = instance.content_object.user.username - if instance.data_type == 'media': - return os.path.join(username, 'formid-media', filename) + if instance.data_type == "media": + return os.path.join(username, "formid-media", filename) - return os.path.join(username, 'docs', filename) + return os.path.join(username, "docs", filename) def save_metadata(metadata_obj): @@ -69,34 +74,35 @@ def save_metadata(metadata_obj): def get_default_content_type(): content_object, created = ContentType.objects.get_or_create( - app_label="logger", model=XFORM_MODEL_NAME) + app_label="logger", model=XFORM_MODEL_NAME + ) return content_object.id -def unique_type_for_form(content_object, - data_type, - data_value=None, - data_file=None): +def unique_type_for_form(content_object, data_type, data_value=None, data_file=None): """ Ensure that each metadata object has unique xform and data_type fields return the metadata object """ - defaults = {'data_value': data_value} if data_value else {} + defaults = {"data_value": data_value} if data_value else {} content_type = ContentType.objects.get_for_model(content_object) if data_value is None and data_file is None: result = MetaData.objects.filter( - object_id=content_object.id, content_type=content_type, - data_type=data_type).first() + object_id=content_object.id, content_type=content_type, data_type=data_type + ).first() else: result, created = MetaData.objects.update_or_create( - object_id=content_object.id, content_type=content_type, - data_type=data_type, defaults=defaults) + object_id=content_object.id, + content_type=content_type, + data_type=data_type, + defaults=defaults, + ) if data_file: - if result.data_value is None or result.data_value == '': + if result.data_value is None or result.data_value == "": result.data_value = data_file.name result.data_file = data_file result.data_file_type = data_file.content_type @@ -107,15 +113,15 @@ def unique_type_for_form(content_object, def type_for_form(content_object, data_type): content_type = ContentType.objects.get_for_model(content_object) - return MetaData.objects.filter(object_id=content_object.id, - content_type=content_type, - data_type=data_type) + return MetaData.objects.filter( + object_id=content_object.id, content_type=content_type, data_type=data_type + ) def create_media(media): """Download media link""" if is_valid_url(media.data_value): - filename = media.data_value.split('/')[-1] + filename = media.data_value.split("/")[-1] data_file = NamedTemporaryFile() content_type = mimetypes.guess_type(filename) with closing(requests.get(media.data_value, stream=True)) as r: @@ -127,8 +133,8 @@ def create_media(media): data_file.seek(os.SEEK_SET) media.data_value = filename media.data_file = InMemoryUploadedFile( - data_file, 'data_file', filename, content_type, - size, charset=None) + data_file, "data_file", filename, content_type, size, charset=None + ) return media @@ -147,7 +153,7 @@ def media_resources(media_list, download=False): """ data = [] for media in media_list: - if media.data_file.name == '' and download: + if media.data_file.name == "" and download: media = create_media(media) if media: @@ -167,17 +173,17 @@ class MetaData(models.Model): date_created = models.DateTimeField(null=True, auto_now_add=True) date_modified = models.DateTimeField(null=True, auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) - content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, - default=get_default_content_type) + content_type = models.ForeignKey( + ContentType, on_delete=models.CASCADE, default=get_default_content_type + ) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") objects = models.Manager() class Meta: - app_label = 'main' - unique_together = ('object_id', 'data_type', 'data_value', - 'content_type') + app_label = "main" + unique_together = ("object_id", "data_type", "data_value", "content_type") def save(self, *args, **kwargs): self._set_hash() @@ -185,7 +191,7 @@ def save(self, *args, **kwargs): @property def hash(self): - if self.file_hash is not None and self.file_hash != '': + if self.file_hash is not None and self.file_hash != "": return self.file_hash else: return self._set_hash() @@ -196,19 +202,19 @@ def _set_hash(self): file_exists = self.data_file.storage.exists(self.data_file.name) - if (file_exists and self.data_file.name != '') \ - or (not file_exists and self.data_file): + if (file_exists and self.data_file.name != "") or ( + not file_exists and self.data_file + ): try: self.data_file.seek(os.SEEK_SET) except IOError: - return '' + return "" else: - self.file_hash = 'md5:%s' % md5( - self.data_file.read()).hexdigest() + self.file_hash = "md5:%s" % md5(self.data_file.read()).hexdigest() return self.file_hash - return '' + return "" def soft_delete(self): """ @@ -222,12 +228,12 @@ def soft_delete(self): @staticmethod def public_link(content_object, data_value=None): - data_type = 'public_link' + data_type = "public_link" if data_value is False: - data_value = 'False' + data_value = "False" metadata = unique_type_for_form(content_object, data_type, data_value) # make text field a boolean - return metadata and metadata.data_value == 'True' + return metadata and metadata.data_value == "True" @staticmethod def set_google_sheet_details(content_object, data_value=None): @@ -243,7 +249,7 @@ def get_google_sheet_details(obj): :param content_object_pk: xform primary key :return dictionary containing google sheet details """ - if isinstance(obj, basestring): + if isinstance(obj, str): metadata_data_value = obj else: metadata = MetaData.objects.filter( @@ -252,57 +258,55 @@ def get_google_sheet_details(obj): metadata_data_value = metadata and metadata.data_value if metadata_data_value: - data_list = metadata_data_value.split('|') + data_list = metadata_data_value.split("|") if data_list: # the data_list format is something like ['A a', 'B b c'] and # the list comprehension and dict cast results to # {'A': 'a', 'B': 'b c'} - return dict( - [tuple(a.strip().split(' ', 1)) for a in data_list]) + return dict([tuple(a.strip().split(" ", 1)) for a in data_list]) @staticmethod def published_by_formbuilder(content_object, data_value=None): - data_type = 'published_by_formbuilder' + data_type = "published_by_formbuilder" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_url(content_object, data_value=None): - data_type = 'enketo_url' + data_type = "enketo_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_preview_url(content_object, data_value=None): - data_type = 'enketo_preview_url' + data_type = "enketo_preview_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_single_submit_url(content_object, data_value=None): - data_type = 'enketo_single_submit_url' + data_type = "enketo_single_submit_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def form_license(content_object, data_value=None): - data_type = 'form_license' + data_type = "form_license" obj = unique_type_for_form(content_object, data_type, data_value) return (obj and obj.data_value) or None @staticmethod def data_license(content_object, data_value=None): - data_type = 'data_license' + data_type = "data_license" obj = unique_type_for_form(content_object, data_type, data_value) return (obj and obj.data_value) or None @staticmethod def source(content_object, data_value=None, data_file=None): - data_type = 'source' - return unique_type_for_form( - content_object, data_type, data_value, data_file) + data_type = "source" + return unique_type_for_form(content_object, data_type, data_value, data_file) @staticmethod def supporting_docs(content_object, data_file=None): - data_type = 'supporting_doc' + data_type = "supporting_doc" if data_file: content_type = ContentType.objects.get_for_model(content_object) @@ -312,24 +316,26 @@ def supporting_docs(content_object, data_file=None): object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_file.content_type - }) + "data_file": data_file, + "data_file_type": data_file.content_type, + }, + ) return type_for_form(content_object, data_type) @staticmethod def media_upload(content_object, data_file=None, download=False): - data_type = 'media' + data_type = "media" if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type in allowed_types: - content_type = ContentType.objects.get_for_model( - content_object) + content_type = ContentType.objects.get_for_model(content_object) media, created = MetaData.objects.update_or_create( data_type=data_type, @@ -337,85 +343,91 @@ def media_upload(content_object, data_file=None, download=False): object_id=content_object.id, data_value=data_file.name, defaults={ - 'data_file': data_file, - 'data_file_type': data_content_type - }) - return media_resources( - type_for_form(content_object, data_type), download) + "data_file": data_file, + "data_file_type": data_content_type, + }, + ) + return media_resources(type_for_form(content_object, data_type), download) @staticmethod def media_add_uri(content_object, uri): """Add a uri as a media resource""" - data_type = 'media' + data_type = "media" if is_valid_url(uri): media, created = MetaData.objects.update_or_create( data_type=data_type, data_value=uri, defaults={ - 'content_object': content_object, - }) + "content_object": content_object, + }, + ) @staticmethod def mapbox_layer_upload(content_object, data=None): - data_type = 'mapbox_layer' - if data and not MetaData.objects.filter(object_id=content_object.id, - data_type='mapbox_layer'): - s = '' + data_type = "mapbox_layer" + if data and not MetaData.objects.filter( + object_id=content_object.id, data_type="mapbox_layer" + ): + s = "" for key in data: - s = s + data[key] + '||' + s = s + data[key] + "||" content_type = ContentType.objects.get_for_model(content_object) - mapbox_layer = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=s) + mapbox_layer = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=s, + ) mapbox_layer.save() if type_for_form(content_object, data_type): - values = type_for_form( - content_object, data_type)[0].data_value.split('||') + values = type_for_form(content_object, data_type)[0].data_value.split("||") data_values = {} - data_values['map_name'] = values[0] - data_values['link'] = values[1] - data_values['attribution'] = values[2] - data_values['id'] = type_for_form(content_object, data_type)[0].id + data_values["map_name"] = values[0] + data_values["link"] = values[1] + data_values["attribution"] = values[2] + data_values["id"] = type_for_form(content_object, data_type)[0].id return data_values else: return None @staticmethod def external_export(content_object, data_value=None): - data_type = 'external_export' + data_type = "external_export" if data_value: content_type = ContentType.objects.get_for_model(content_object) - result = MetaData(data_type=data_type, - content_type=content_type, - object_id=content_object.id, - data_value=data_value) + result = MetaData( + data_type=data_type, + content_type=content_type, + object_id=content_object.id, + data_value=data_value, + ) result.save() return result return MetaData.objects.filter( - object_id=content_object.id, data_type=data_type).order_by('-id') + object_id=content_object.id, data_type=data_type + ).order_by("-id") @property def external_export_url(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") return parts[1] if len(parts) > 1 else None @property def external_export_name(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") return parts[0] if len(parts) > 1 else None @property def external_export_template(self): - parts = self.data_value.split('|') + parts = self.data_value.split("|") - return parts[1].replace('xls', 'templates') if len(parts) > 1 else None + return parts[1].replace("xls", "templates") if len(parts) > 1 else None @staticmethod def textit(content_object, data_value=None): @@ -432,34 +444,32 @@ def textit_flow_details(content_object, data_value: str = ""): @property def is_linked_dataset(self): - return ( - isinstance(self.data_value, basestring) and - (self.data_value.startswith('xform') or - self.data_value.startswith('dataview')) + return isinstance(self.data_value, str) and ( + self.data_value.startswith("xform") + or self.data_value.startswith("dataview") ) @staticmethod def xform_meta_permission(content_object, data_value=None): data_type = XFORM_META_PERMS - return unique_type_for_form( - content_object, data_type, data_value) + return unique_type_for_form(content_object, data_type, data_value) @staticmethod def submission_review(content_object, data_value=None): - data_type = 'submission_review' + data_type = "submission_review" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def instance_csv_imported_by(content_object, data_value=None): - data_type = 'imported_via_csv_by' + data_type = "imported_via_csv_by" return unique_type_for_form(content_object, data_type, data_value) def clear_cached_metadata_instance_object( - sender, instance=None, created=False, **kwargs): - safe_delete('{}{}'.format( - XFORM_METADATA_CACHE, instance.object_id)) + sender, instance=None, created=False, **kwargs +): + safe_delete("{}{}".format(XFORM_METADATA_CACHE, instance.object_id)) def update_attached_object(sender, instance=None, created=False, **kwargs): @@ -467,9 +477,16 @@ def update_attached_object(sender, instance=None, created=False, **kwargs): instance.content_object.save() -post_save.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_object') -post_save.connect(update_attached_object, sender=MetaData, - dispatch_uid='update_attached_xform') -post_delete.connect(clear_cached_metadata_instance_object, sender=MetaData, - dispatch_uid='clear_cached_metadata_instance_delete') +post_save.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_object", +) +post_save.connect( + update_attached_object, sender=MetaData, dispatch_uid="update_attached_xform" +) +post_delete.connect( + clear_cached_metadata_instance_object, + sender=MetaData, + dispatch_uid="clear_cached_metadata_instance_delete", +) diff --git a/onadata/apps/main/tests/test_form_enter_data.py b/onadata/apps/main/tests/test_form_enter_data.py index b27dbc51f4..27b1ebca23 100644 --- a/onadata/apps/main/tests/test_form_enter_data.py +++ b/onadata/apps/main/tests/test_form_enter_data.py @@ -10,7 +10,6 @@ from future.moves.urllib.parse import urlparse from httmock import HTTMock, urlmatch from nose import SkipTest -from past.builtins import basestring from onadata.apps.logger.views import enter_data from onadata.apps.main.models import MetaData @@ -39,9 +38,7 @@ def enketo_mock_http(url, request): def enketo_error_mock(url, request): response = requests.Response() response.status_code = 400 - response._content = ( - '{"message": "no account exists for this OpenRosa server"}' - ) + response._content = '{"message": "no account exists for this OpenRosa server"}' return response @@ -80,8 +77,8 @@ def test_enketo_remote_server(self): server_url = "https://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def test_enketo_url_with_http_protocol_on_formlist(self): if not self._running_enketo(): @@ -90,9 +87,9 @@ def test_enketo_url_with_http_protocol_on_formlist(self): server_url = "http://testserver.com/bob" form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time())) # noqa url = get_enketo_urls(server_url, form_id) - self.assertIn("http:", url['url']) - self.assertIsInstance(url['url'], basestring) - self.assertIsNone(URLValidator()(url['url'])) + self.assertIn("http:", url["url"]) + self.assertIsInstance(url["url"], str) + self.assertIsNone(URLValidator()(url["url"])) def _get_grcode_view_response(self): factory = RequestFactory() @@ -105,9 +102,7 @@ def _get_grcode_view_response(self): def test_qrcode_view(self): with HTTMock(enketo_mock): response = self._get_grcode_view_response() - self.assertContains( - response, "data:image/png;base64,", status_code=200 - ) + self.assertContains(response, "data:image/png;base64,", status_code=200) def test_qrcode_view_with_enketo_error(self): with HTTMock(enketo_error_mock): @@ -121,9 +116,7 @@ def test_enter_data_redir(self): factory = RequestFactory() request = factory.get("/") request.user = self.user - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) # make sure response redirect to an enketo site enketo_base_url = urlparse(settings.ENKETO_URL).netloc redirected_base_url = urlparse(response["Location"]).netloc @@ -157,9 +150,7 @@ def test_public_with_link_to_share_toggle_on(self): factory = RequestFactory() request = factory.get("/") request.user = AnonymousUser() - response = enter_data( - request, self.user.username, self.xform.id_string - ) + response = enter_data(request, self.user.username, self.xform.id_string) self.assertEqual(response.status_code, 302) def test_enter_data_non_existent_user(self): diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index f6af0471f8..cd3c1a58d5 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -11,7 +11,6 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.http import HttpRequest from django.utils.translation import ugettext as _ -from past.builtins import basestring from onadata.apps.logger.models import XForm from onadata.apps.logger.models.instance import FormInactiveError @@ -22,42 +21,49 @@ from onadata.libs.utils.log import audit_log from onadata.libs.utils.logger_tools import create_instance -SMS_API_ERROR = 'SMS_API_ERROR' -SMS_PARSING_ERROR = 'SMS_PARSING_ERROR' -SMS_SUBMISSION_ACCEPTED = 'SMS_SUBMISSION_ACCEPTED' -SMS_SUBMISSION_REFUSED = 'SMS_SUBMISSION_REFUSED' -SMS_INTERNAL_ERROR = 'SMS_INTERNAL_ERROR' +SMS_API_ERROR = "SMS_API_ERROR" +SMS_PARSING_ERROR = "SMS_PARSING_ERROR" +SMS_SUBMISSION_ACCEPTED = "SMS_SUBMISSION_ACCEPTED" +SMS_SUBMISSION_REFUSED = "SMS_SUBMISSION_REFUSED" +SMS_INTERNAL_ERROR = "SMS_INTERNAL_ERROR" -BASE64_ALPHABET = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' - 'abcdefghijklmnopqrstuvwxyz0123456789+/=') -DEFAULT_SEPARATOR = '+' +BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz0123456789+/=" +DEFAULT_SEPARATOR = "+" DEFAULT_ALLOW_MEDIA = False -NA_VALUE = 'n/a' +NA_VALUE = "n/a" BASE64_ALPHABET = None -META_FIELDS = ('start', 'end', 'today', 'deviceid', 'subscriberid', - 'imei', 'phonenumber') -MEDIA_TYPES = ('audio', 'video', 'photo') -DEFAULT_DATE_FORMAT = '%Y-%m-%d' -DEFAULT_DATETIME_FORMAT = '%Y-%m-%d-%H:%M' -SENSITIVE_FIELDS = ('text', 'select all that apply', 'geopoint', 'barcode') +META_FIELDS = ( + "start", + "end", + "today", + "deviceid", + "subscriberid", + "imei", + "phonenumber", +) +MEDIA_TYPES = ("audio", "video", "photo") +DEFAULT_DATE_FORMAT = "%Y-%m-%d" +DEFAULT_DATETIME_FORMAT = "%Y-%m-%d-%H:%M" +SENSITIVE_FIELDS = ("text", "select all that apply", "geopoint", "barcode") def is_last(index, items): - return index == len(items) - 1 or (items[-1].get('type') == 'note' and - index == len(items) - 2) + return index == len(items) - 1 or ( + items[-1].get("type") == "note" and index == len(items) - 2 + ) def get_sms_instance_id(instance): - """ Human-friendly unique ID of a submission for latter ref/update + """Human-friendly unique ID of a submission for latter ref/update - For now, we strip down to the first 8 chars of the UUID. - Until we figure out what we really want (might as well be used - by formhub XML) """ + For now, we strip down to the first 8 chars of the UUID. + Until we figure out what we really want (might as well be used + by formhub XML)""" return instance.uuid[:8] def sms_media_to_file(file_object, name): - if isinstance(file_object, basestring): + if isinstance(file_object, str): file_object = io.BytesIO(file_object) def getsize(f): @@ -70,90 +76,92 @@ def getsize(f): name = name.strip() content_type, charset = mimetypes.guess_type(name) size = getsize(file_object) - return InMemoryUploadedFile(file=file_object, name=name, - field_name=None, content_type=content_type, - charset=charset, size=size) + return InMemoryUploadedFile( + file=file_object, + name=name, + field_name=None, + content_type=content_type, + charset=charset, + size=size, + ) def generate_instance(username, xml_file, media_files, uuid=None): - ''' Process an XForm submission as if done via HTTP + """Process an XForm submission as if done via HTTP - :param IO xml_file: file-like object containing XML XForm - :param string username: username of the Form's owner - :param list media_files: a list of UploadedFile objects - :param string uuid: an optionnal uuid for the instance. + :param IO xml_file: file-like object containing XML XForm + :param string username: username of the Form's owner + :param list media_files: a list of UploadedFile objects + :param string uuid: an optionnal uuid for the instance. - :returns a (status, message) tuple. ''' + :returns a (status, message) tuple.""" try: - instance = create_instance( - username, - xml_file, - media_files, - uuid=uuid - ) + instance = create_instance(username, xml_file, media_files, uuid=uuid) except InstanceInvalidUserError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Username or ID required.")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Username or ID required.")} except InstanceEmptyError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Received empty submission. " - u"No instance was created")} + return { + "code": SMS_INTERNAL_ERROR, + "text": _("Received empty submission. " "No instance was created"), + } except FormInactiveError: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form is not active")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Form is not active")} except XForm.DoesNotExist: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Form does not exist on this account")} + return { + "code": SMS_SUBMISSION_REFUSED, + "text": _("Form does not exist on this account"), + } except ExpatError: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Improperly formatted XML.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Improperly formatted XML.")} except DuplicateInstance: - return {'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Duplicate submission")} + return {"code": SMS_SUBMISSION_REFUSED, "text": _("Duplicate submission")} if instance is None: - return {'code': SMS_INTERNAL_ERROR, - 'text': _(u"Unable to create submission.")} + return {"code": SMS_INTERNAL_ERROR, "text": _("Unable to create submission.")} user = User.objects.get(username=username) - audit = { - "xform": instance.xform.id_string - } - audit_log(Actions.SUBMISSION_CREATED, - user, instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, HttpRequest()) + audit = {"xform": instance.xform.id_string} + audit_log( + Actions.SUBMISSION_CREATED, + user, + instance.xform.user, + _("Created submission on form %(id_string)s.") + % {"id_string": instance.xform.id_string}, + audit, + HttpRequest(), + ) xml_file.close() if len(media_files): [_file.close() for _file in media_files] - return {'code': SMS_SUBMISSION_ACCEPTED, - 'text': _(u"[SUCCESS] Your submission has been accepted."), - 'id': get_sms_instance_id(instance)} + return { + "code": SMS_SUBMISSION_ACCEPTED, + "text": _("[SUCCESS] Your submission has been accepted."), + "id": get_sms_instance_id(instance), + } def is_sms_related(json_survey): - ''' Whether a form is considered to want sms Support + """Whether a form is considered to want sms Support - return True if one sms-related field is defined. ''' + return True if one sms-related field is defined.""" def treat(value, key=None): if key is None: return False - if key in ('sms_field', 'sms_option') and value: - if not value.lower() in ('no', 'false'): + if key in ("sms_field", "sms_option") and value: + if not value.lower() in ("no", "false"): return True def walk(dl): if not isinstance(dl, (dict, list)): return False - iterator = [(None, e) for e in dl] \ - if isinstance(dl, list) else dl.items() + iterator = [(None, e) for e in dl] if isinstance(dl, list) else dl.items() for k, v in iterator: - if k == 'parent': + if k == "parent": continue if treat(v, k): return True @@ -165,86 +173,100 @@ def walk(dl): def check_form_sms_compatibility(form, json_survey=None): - ''' Tests all SMS related rules on the XForm representation + """Tests all SMS related rules on the XForm representation - Returns a view-compatible dict(type, text) with warnings or - a success message ''' + Returns a view-compatible dict(type, text) with warnings or + a success message""" if json_survey is None: - json_survey = form.get('form_o', {}) + json_survey = form.get("form_o", {}) def prep_return(msg, comp=None): from django.urls import reverse - error = 'alert-info' - warning = 'alert-info' - success = 'alert-success' - outro = (u"
    Please check the " - u"SMS Syntax Page." % {'syntax_url': reverse('syntax')}) + error = "alert-info" + warning = "alert-info" + success = "alert-success" + outro = ( + '
    Please check the ' + "SMS Syntax Page." % {"syntax_url": reverse("syntax")} + ) # no compatibility at all if not comp: alert = error - msg = (u"%(prefix)s %(msg)s" - % {'prefix': u"Your Form is not SMS-compatible" - u". If you want to later enable " - u"SMS Support, please fix:
    ", - 'msg': msg}) + msg = "%(prefix)s %(msg)s" % { + "prefix": "Your Form is not SMS-compatible" + ". If you want to later enable " + "SMS Support, please fix:
    ", + "msg": msg, + } # no blocker but could be improved elif comp == 1: alert = warning - msg = (u"%(prefix)s
      %(msg)s
    " - % {'prefix': u"Your form can be used with SMS, " - u"knowing that:", 'msg': msg}) + msg = "%(prefix)s
      %(msg)s
    " % { + "prefix": "Your form can be used with SMS, " "knowing that:", + "msg": msg, + } # SMS compatible else: - outro = u"" + outro = "" alert = success - return {'type': alert, - 'text': u"%(msg)s%(outro)s" - % {'msg': msg, 'outro': outro}} + return { + "type": alert, + "text": "%(msg)s%(outro)s" % {"msg": msg, "outro": outro}, + } # first level children. should be groups - groups = json_survey.get('children', [{}]) + groups = json_survey.get("children", [{}]) # BLOCKERS # overload SENSITIVE_FIELDS if date or datetime format contain spaces. sensitive_fields = copy.copy(SENSITIVE_FIELDS) - date_format = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - datetime_format = json_survey.get('sms_datetime_format', - DEFAULT_DATETIME_FORMAT) \ + date_format = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + datetime_format = ( + json_survey.get("sms_datetime_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) if len(date_format.split()) > 1: - sensitive_fields += ('date', ) + sensitive_fields += ("date",) if len(datetime_format.split()) > 1: - sensitive_fields += ('datetime', ) + sensitive_fields += ("datetime",) # must not contain out-of-group questions - if sum([1 for e in groups if e.get('type') != 'group']): - return prep_return(_(u"All your questions must be in groups.")) + if sum([1 for e in groups if e.get("type") != "group"]): + return prep_return(_("All your questions must be in groups.")) # all groups must have an sms_field - bad_groups = [e.get('name') for e in groups - if not e.get('sms_field', '') and - not e.get('name', '') == 'meta'] + bad_groups = [ + e.get("name") + for e in groups + if not e.get("sms_field", "") and not e.get("name", "") == "meta" + ] if len(bad_groups): - return prep_return(_(u"All your groups must have an 'sms_field' " - u"(use 'meta' prefixed ones for non-fillable " - u"groups). %s" % bad_groups[-1])) + return prep_return( + _( + "All your groups must have an 'sms_field' " + "(use 'meta' prefixed ones for non-fillable " + "groups). %s" % bad_groups[-1] + ) + ) # all select_one or select_multiple fields muts have sms_option for each. for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) for field in fields: - xlsf_type = field.get('type') - xlsf_name = field.get('name') - xlsf_choices = field.get('children') - if xlsf_type in ('select one', 'select all that apply'): + xlsf_type = field.get("type") + xlsf_name = field.get("name") + xlsf_choices = field.get("children") + if xlsf_type in ("select one", "select all that apply"): nb_choices = len(xlsf_choices) - options = list(set([c.get('sms_option', '') or None - for c in xlsf_choices])) + options = list( + set([c.get("sms_option", "") or None for c in xlsf_choices]) + ) try: options.remove(None) except ValueError: @@ -252,75 +274,93 @@ def prep_return(msg, comp=None): nb_options = len(options) if nb_choices != nb_options: return prep_return( - _(u"Not all options in the choices list for " - u"%s have an " - u"sms_option value.") % xlsf_name + _( + "Not all options in the choices list for " + "%s have an " + "sms_option value." + ) + % xlsf_name ) # has sensitive (space containing) fields in non-last position for group in groups: - fields = group.get('children', [{}]) + fields = group.get("children", [{}]) last_pos = len(fields) - 1 # consider last field to be last before note if there's a trailing note - if fields[last_pos].get('type') == 'note': + if fields[last_pos].get("type") == "note": if len(fields) - 1: last_pos -= 1 for idx, field in enumerate(fields): - if idx != last_pos and field.get('type', '') in sensitive_fields: - return prep_return(_(u"Questions for which values can contain " - u"spaces are only allowed on last " - u"position of group (%s)" - % field.get('name'))) + if idx != last_pos and field.get("type", "") in sensitive_fields: + return prep_return( + _( + "Questions for which values can contain " + "spaces are only allowed on last " + "position of group (%s)" % field.get("name") + ) + ) # separator is not set or is within BASE64 alphabet and sms_allow_media - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR - sms_allow_media = bool(json_survey.get('sms_allow_media', - DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA) + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR + sms_allow_media = bool( + json_survey.get("sms_allow_media", DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA + ) if sms_allow_media and separator in BASE64_ALPHABET: - return prep_return(_(u"When allowing medias ('sms_allow_media'), your " - u"separator (%s) must be outside Base64 alphabet " - u"(letters, digits and +/=). " - u"You case use '#' instead." % separator)) + return prep_return( + _( + "When allowing medias ('sms_allow_media'), your " + "separator (%s) must be outside Base64 alphabet " + "(letters, digits and +/=). " + "You case use '#' instead." % separator + ) + ) # WARNINGS warnings = [] # sms_separator not set - if not json_survey.get('sms_separator', ''): - warnings.append(u"
  • You have not set a separator. Default '+' " - u"separator will be used.
    • ") + if not json_survey.get("sms_separator", ""): + warnings.append( + "
  • You have not set a separator. Default '+' " + "separator will be used.
    • " + ) # has date field with no sms_date_format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'date']): - warnings.append(u"
  • You have 'date' fields without " - u"explicitly setting a date format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATE_FORMAT) + if sum([1 for e in group.get("children", [{}]) if e.get("type") == "date"]): + warnings.append( + "
  • You have 'date' fields without " + "explicitly setting a date format. " + "Default (%s) will be used.
    • " % DEFAULT_DATE_FORMAT + ) break # has datetime field with no datetime format - if not json_survey.get('sms_date_format', ''): + if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get('children', [{}]) - if e.get('type') == 'datetime']): - warnings.append(u"
  • You have 'datetime' fields without " - u"explicitly setting a datetime format. " - u"Default (%s) will be used.
    • " - % DEFAULT_DATETIME_FORMAT) + if sum( + [1 for e in group.get("children", [{}]) if e.get("type") == "datetime"] + ): + warnings.append( + "
  • You have 'datetime' fields without " + "explicitly setting a datetime format. " + "Default (%s) will be used.
    • " % DEFAULT_DATETIME_FORMAT + ) break # date or datetime format contain space - if 'date' in sensitive_fields: - warnings.append(u"
  • 'sms_date_format' contains space which will " - u"require 'date' questions to be positioned at " - u"the end of groups (%s).
    • " % date_format) - if 'datetime' in sensitive_fields: - warnings.append(u"
  • 'sms_datetime_format' contains space which will " - u"require 'datetime' questions to be positioned at " - u"the end of groups (%s).
    • " % datetime_format) + if "date" in sensitive_fields: + warnings.append( + "
  • 'sms_date_format' contains space which will " + "require 'date' questions to be positioned at " + "the end of groups (%s).
    • " % date_format + ) + if "datetime" in sensitive_fields: + warnings.append( + "
  • 'sms_datetime_format' contains space which will " + "require 'datetime' questions to be positioned at " + "the end of groups (%s).
    • " % datetime_format + ) if len(warnings): - return prep_return(u"".join(warnings), comp=1) + return prep_return("".join(warnings), comp=1) # Good to go - return prep_return(_(u"Note that your form is also SMS comptatible."), 2) + return prep_return(_("Note that your form is also SMS comptatible."), 2) diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 63992c77aa..3193b1a85c 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,17 +1,15 @@ -from past.builtins import basestring - import django.dispatch from onadata.apps.logger.models import XForm -xform_tags_add = django.dispatch.Signal(providing_args=['xform', 'tags']) -xform_tags_delete = django.dispatch.Signal(providing_args=['xform', 'tag']) +xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) +xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) @django.dispatch.receiver(xform_tags_add, sender=XForm) def add_tags_to_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tags = kwargs.get('tags', None) + xform = kwargs.get("xform", None) + tags = kwargs.get("tags", None) if isinstance(xform, XForm) and isinstance(tags, list): # update existing instances with the new tag for instance in xform.instances.all(): @@ -24,9 +22,9 @@ def add_tags_to_xform_instances(sender, **kwargs): @django.dispatch.receiver(xform_tags_delete, sender=XForm) def delete_tag_from_xform_instances(sender, **kwargs): - xform = kwargs.get('xform', None) - tag = kwargs.get('tag', None) - if isinstance(xform, XForm) and isinstance(tag, basestring): + xform = kwargs.get("xform", None) + tag = kwargs.get("tag", None) + if isinstance(xform, XForm) and isinstance(tag, str): # update existing instances with the new tag for instance in xform.instances.all(): if tag in instance.tags.names(): diff --git a/onadata/libs/serializers/chart_serializer.py b/onadata/libs/serializers/chart_serializer.py index a35ab22765..69b1f7b41d 100644 --- a/onadata/libs/serializers/chart_serializer.py +++ b/onadata/libs/serializers/chart_serializer.py @@ -1,5 +1,3 @@ -from past.builtins import basestring - from django.http import Http404 from rest_framework import serializers @@ -11,37 +9,36 @@ class ChartSerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( - view_name='chart-detail', lookup_field='pk') + view_name="chart-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") class FieldsChartSerializer(serializers.ModelSerializer): - class Meta: model = XForm def to_representation(self, obj): data = {} - request = self.context.get('request') + request = self.context.get("request") if obj is not None: fields = obj.survey_elements if request: - selected_fields = request.query_params.get('fields') + selected_fields = request.query_params.get("fields") - if isinstance(selected_fields, basestring) \ - and selected_fields != 'all': - fields = selected_fields.split(',') - fields = [e for e in obj.survey_elements - if e.name in fields] + if isinstance(selected_fields, str) and selected_fields != "all": + fields = selected_fields.split(",") + fields = [e for e in obj.survey_elements if e.name in fields] if len(fields) == 0: raise Http404( - "Field %s does not not exist on the form" % fields) + "Field %s does not not exist on the form" % fields + ) for field in fields: if field.name == INSTANCE_ID: diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index 72b9253d1c..75f54d3f5e 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -1,18 +1,16 @@ import json from builtins import str as text -from past.builtins import basestring from rest_framework import serializers class JsonField(serializers.Field): - def to_representation(self, value): - if isinstance(value, basestring): + if isinstance(value, str): return json.loads(value) return value def to_internal_value(self, value): - if isinstance(value, basestring): + if isinstance(value, str): try: return json.loads(value) except ValueError as e: @@ -22,6 +20,6 @@ def to_internal_value(self, value): @classmethod def to_json(cls, data): - if isinstance(data, basestring): + if isinstance(data, str): return json.loads(data) return data diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index f0c4c710ba..6c0a78f837 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -2,7 +2,6 @@ """ Organization Serializer """ -from past.builtins import basestring # pylint: disable=redefined-builtin from django.contrib.auth.models import User from django.db.models.query import QuerySet @@ -12,9 +11,11 @@ from onadata.apps.api import tools from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (_get_first_last_names, - get_organization_members, - get_organization_owners) +from onadata.apps.api.tools import ( + _get_first_last_names, + get_organization_members, + get_organization_owners, +) from onadata.apps.main.forms import RegistrationFormUserProfile from onadata.libs.permissions import get_role_in_org from onadata.libs.serializers.fields.json_field import JsonField @@ -24,64 +25,64 @@ class OrganizationSerializer(serializers.HyperlinkedModelSerializer): """ Organization profile serializer """ + url = serializers.HyperlinkedIdentityField( - view_name='organizationprofile-detail', lookup_field='user') - org = serializers.CharField(source='user.username', max_length=30) + view_name="organizationprofile-detail", lookup_field="user" + ) + org = serializers.CharField(source="user.username", max_length=30) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) creator = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) users = serializers.SerializerMethodField() metadata = JsonField(required=False) name = serializers.CharField(max_length=30) class Meta: model = OrganizationProfile - exclude = ('created_by', 'is_organization', 'organization') - owner_only_fields = ('metadata', ) + exclude = ("created_by", "is_organization", "organization") + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): super(OrganizationSerializer, self).__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") is_permitted = ( - request and request.user and - request.user.has_perm('api.view_organizationprofile', - self.instance)) - if isinstance(self.instance, QuerySet) or not is_permitted or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + request + and request.user + and request.user.has_perm("api.view_organizationprofile", self.instance) + ) + if isinstance(self.instance, QuerySet) or not is_permitted or not request: + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def update(self, instance, validated_data): # update the user model - if 'name' in validated_data: - first_name, last_name = \ - _get_first_last_names(validated_data.get('name')) + if "name" in validated_data: + first_name, last_name = _get_first_last_names(validated_data.get("name")) instance.user.first_name = first_name instance.user.last_name = last_name instance.user.save() - return super(OrganizationSerializer, self).update( - instance, validated_data - ) + return super(OrganizationSerializer, self).update(instance, validated_data) def create(self, validated_data): - org = validated_data.get('user') + org = validated_data.get("user") if org: - org = org.get('username') + org = org.get("username") - org_name = validated_data.get('name', None) + org_name = validated_data.get("name", None) creator = None - if 'request' in self.context: - creator = self.context['request'].user + if "request" in self.context: + creator = self.context["request"].user - validated_data['organization'] = org_name + validated_data["organization"] = org_name - profile = tools.create_organization_object(org, creator, - validated_data) + profile = tools.create_organization_object(org, creator, validated_data) profile.save() return profile @@ -90,45 +91,48 @@ def validate_org(self, value): # pylint: disable=no-self-use """ Validate organization name. """ - org = value.lower() if isinstance(value, basestring) else value + org = value.lower() if isinstance(value, str) else value if org in RegistrationFormUserProfile.RESERVED_USERNAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % org - )) + raise serializers.ValidationError( + _("%s is a reserved name, please choose another" % org) + ) elif not RegistrationFormUserProfile.legal_usernames_re.search(org): - raise serializers.ValidationError(_( - u"Organization may only contain alpha-numeric characters and " - u"underscores" - )) + raise serializers.ValidationError( + _( + "Organization may only contain alpha-numeric characters and " + "underscores" + ) + ) try: User.objects.get(username=org) except User.DoesNotExist: return org - raise serializers.ValidationError(_( - u"Organization %s already exists." % org - )) + raise serializers.ValidationError(_("Organization %s already exists." % org)) def get_users(self, obj): # pylint: disable=no-self-use """ Return organization members. """ + def create_user_list(user_list): - return [{ - 'user': u.username, - 'role': get_role_in_org(u, obj), - 'first_name': u.first_name, - 'last_name': u.last_name, - 'gravatar': u.profile.gravatar - } for u in user_list] + return [ + { + "user": u.username, + "role": get_role_in_org(u, obj), + "first_name": u.first_name, + "last_name": u.last_name, + "gravatar": u.profile.gravatar, + } + for u in user_list + ] members = get_organization_members(obj) if obj else [] owners = get_organization_owners(obj) if obj else [] if owners and members: - members = members.exclude( - username__in=[user.username for user in owners]) + members = members.exclude(username__in=[user.username for user in owners]) members_list = create_user_list(members) owners_list = create_user_list(owners) diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index f2686c4131..10570f9f16 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -5,7 +5,6 @@ import copy import re -from past.builtins import basestring # pylint: disable=redefined-builtin from django.conf import settings from django.contrib.auth.models import User @@ -30,9 +29,7 @@ from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.utils.cache_tools import IS_ORG from onadata.libs.utils.analytics import track_object_event -from onadata.libs.utils.email import ( - get_verification_url, get_verification_email_data -) +from onadata.libs.utils.email import get_verification_url, get_verification_email_data RESERVED_NAMES = RegistrationFormUserProfile.RESERVED_USERNAMES LEGAL_USERNAMES_REGEX = RegistrationFormUserProfile.legal_usernames_re @@ -47,59 +44,55 @@ def _get_first_last_names(name, limit=30): # imposition of 30 characters on both first_name and last_name hence # ensure we only have 30 characters for either field - return name[:limit], name[limit:limit * 2] + return name[:limit], name[limit : limit * 2] name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_registration_params(attrs): params = copy.deepcopy(attrs) - name = params.get('name', None) - user = params.pop('user', None) + name = params.get("name", None) + user = params.pop("user", None) if user: - username = user.pop('username', None) - password = user.pop('password', None) - first_name = user.pop('first_name', None) - last_name = user.pop('last_name', None) - email = user.pop('email', None) + username = user.pop("username", None) + password = user.pop("password", None) + first_name = user.pop("first_name", None) + last_name = user.pop("last_name", None) + email = user.pop("email", None) if username: - params['username'] = username + params["username"] = username if email: - params['email'] = email + params["email"] = email if password: - params.update({'password1': password, 'password2': password}) + params.update({"password1": password, "password2": password}) if first_name: - params['first_name'] = first_name + params["first_name"] = first_name - params['last_name'] = last_name or '' + params["last_name"] = last_name or "" # For backward compatibility, Users who still use only name if name: - first_name, last_name = \ - _get_first_last_names(name) - params['first_name'] = first_name - params['last_name'] = last_name + first_name, last_name = _get_first_last_names(name) + params["first_name"] = first_name + params["last_name"] = last_name return params def _send_verification_email(redirect_url, user, request): - verification_key = (user.registrationprofile - .create_new_activation_key()) - verification_url = get_verification_url( - redirect_url, request, verification_key - ) + verification_key = user.registrationprofile.create_new_activation_key() + verification_url = get_verification_url(redirect_url, request, verification_key) email_data = get_verification_email_data( user.email, user.username, verification_url, request @@ -112,47 +105,72 @@ class UserProfileSerializer(serializers.HyperlinkedModelSerializer): """ UserProfile serializer. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', lookup_field='user') + view_name="userprofile-detail", lookup_field="user" + ) is_org = serializers.SerializerMethodField() - username = serializers.CharField(source='user.username', min_length=3, - max_length=30) + username = serializers.CharField( + source="user.username", min_length=3, max_length=30 + ) name = serializers.CharField(required=False, allow_blank=True) - first_name = serializers.CharField(source='user.first_name', - required=False, allow_blank=True, - max_length=30) - last_name = serializers.CharField(source='user.last_name', - required=False, allow_blank=True, - max_length=30) - email = serializers.EmailField(source='user.email') - website = serializers.CharField(source='home_page', required=False, - allow_blank=True) + first_name = serializers.CharField( + source="user.first_name", required=False, allow_blank=True, max_length=30 + ) + last_name = serializers.CharField( + source="user.last_name", required=False, allow_blank=True, max_length=30 + ) + email = serializers.EmailField(source="user.email") + website = serializers.CharField( + source="home_page", required=False, allow_blank=True + ) twitter = serializers.CharField(required=False, allow_blank=True) gravatar = serializers.ReadOnlyField() - password = serializers.CharField(source='user.password', allow_blank=True, - required=False) + password = serializers.CharField( + source="user.password", allow_blank=True, required=False + ) user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) - id = serializers.ReadOnlyField(source='user.id') # pylint: disable=C0103 - joined_on = serializers.ReadOnlyField(source='user.date_joined') + id = serializers.ReadOnlyField(source="user.id") # pylint: disable=C0103 + joined_on = serializers.ReadOnlyField(source="user.date_joined") class Meta: model = UserProfile - fields = ('id', 'is_org', 'url', 'username', 'password', 'first_name', - 'last_name', 'email', 'city', 'country', 'organization', - 'website', 'twitter', 'gravatar', 'require_auth', 'user', - 'metadata', 'joined_on', 'name') - owner_only_fields = ('metadata', ) + fields = ( + "id", + "is_org", + "url", + "username", + "password", + "first_name", + "last_name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "metadata", + "joined_on", + "name", + ) + owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): super(UserProfileSerializer, self).__init__(*args, **kwargs) - if self.instance and hasattr(self.Meta, 'owner_only_fields'): - request = self.context.get('request') - if isinstance(self.instance, QuerySet) or \ - (request and request.user != self.instance.user) or \ - not request: - for field in getattr(self.Meta, 'owner_only_fields'): + if self.instance and hasattr(self.Meta, "owner_only_fields"): + request = self.context.get("request") + if ( + isinstance(self.instance, QuerySet) + or (request and request.user != self.instance.user) + or not request + ): + for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) def get_is_org(self, obj): # pylint: disable=no-self-use @@ -160,12 +178,12 @@ def get_is_org(self, obj): # pylint: disable=no-self-use Returns True if it is an organization profile. """ if obj: - is_org = cache.get('{}{}'.format(IS_ORG, obj.pk)) + is_org = cache.get("{}{}".format(IS_ORG, obj.pk)) if is_org: return is_org is_org = is_organization(obj) - cache.set('{}{}'.format(IS_ORG, obj.pk), is_org) + cache.set("{}{}".format(IS_ORG, obj.pk), is_org) return is_org def to_representation(self, instance): @@ -173,46 +191,46 @@ def to_representation(self, instance): Serialize objects -> primitives. """ ret = super(UserProfileSerializer, self).to_representation(instance) - if 'password' in ret: - del ret['password'] + if "password" in ret: + del ret["password"] - request = self.context['request'] \ - if 'request' in self.context else None + request = self.context["request"] if "request" in self.context else None - if 'email' in ret and request is None or request.user \ - and not request.user.has_perm(CAN_VIEW_PROFILE, instance): - del ret['email'] + if ( + "email" in ret + and request is None + or request.user + and not request.user.has_perm(CAN_VIEW_PROFILE, instance) + ): + del ret["email"] - if 'first_name' in ret: - ret['name'] = u' '.join([ret.get('first_name'), - ret.get('last_name', "")]) - ret['name'] = ret['name'].strip() + if "first_name" in ret: + ret["name"] = " ".join([ret.get("first_name"), ret.get("last_name", "")]) + ret["name"] = ret["name"].strip() return ret def update(self, instance, validated_data): params = validated_data password = params.get("password1") - email = params.get('email') + email = params.get("email") # Check password if email is being updated if email and not password: raise serializers.ValidationError( - _(u'Your password is required when updating your email ' - u'address.')) + _("Your password is required when updating your email " "address.") + ) if password and not instance.user.check_password(password): - raise serializers.ValidationError(_(u'Invalid password')) + raise serializers.ValidationError(_("Invalid password")) # get user instance.user.email = email or instance.user.email - instance.user.first_name = params.get('first_name', - instance.user.first_name) + instance.user.first_name = params.get("first_name", instance.user.first_name) - instance.user.last_name = params.get('last_name', - instance.user.last_name) + instance.user.last_name = params.get("last_name", instance.user.last_name) - instance.user.username = params.get('username', instance.user.username) + instance.user.username = params.get("username", instance.user.username) instance.user.save() @@ -220,8 +238,8 @@ def update(self, instance, validated_data): instance.metadata.update({"is_email_verified": False}) instance.save() - request = self.context.get('request') - redirect_url = params.get('redirect_url') + request = self.context.get("request") + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, instance.user, request) if password: @@ -231,51 +249,48 @@ def update(self, instance, validated_data): return super(UserProfileSerializer, self).update(instance, params) @track_object_event( - user_field='user', - properties={ - 'name': 'name', - 'country': 'country'}) + user_field="user", properties={"name": "name", "country": "country"} + ) def create(self, validated_data): params = validated_data - request = self.context.get('request') + request = self.context.get("request") metadata = {} site = Site.objects.get(pk=settings.SITE_ID) try: new_user = RegistrationProfile.objects.create_inactive_user( - username=params.get('username'), - password=params.get('password1'), - email=params.get('email'), + username=params.get("username"), + password=params.get("password1"), + email=params.get("email"), site=site, - send_email=settings.SEND_EMAIL_ACTIVATION_API) + send_email=settings.SEND_EMAIL_ACTIVATION_API, + ) except IntegrityError: - raise serializers.ValidationError(_( - u"User account {} already exists".format( - params.get('username')) - )) + raise serializers.ValidationError( + _("User account {} already exists".format(params.get("username"))) + ) new_user.is_active = True - new_user.first_name = params.get('first_name') - new_user.last_name = params.get('last_name') + new_user.first_name = params.get("first_name") + new_user.last_name = params.get("last_name") new_user.save() - if getattr( - settings, 'ENABLE_EMAIL_VERIFICATION', False - ): - redirect_url = params.get('redirect_url') + if getattr(settings, "ENABLE_EMAIL_VERIFICATION", False): + redirect_url = params.get("redirect_url") _send_verification_email(redirect_url, new_user, request) created_by = request.user created_by = None if created_by.is_anonymous else created_by - metadata['last_password_edit'] = timezone.now().isoformat() + metadata["last_password_edit"] = timezone.now().isoformat() profile = UserProfile( - user=new_user, name=params.get('first_name'), + user=new_user, + name=params.get("first_name"), created_by=created_by, - city=params.get('city', u''), - country=params.get('country', u''), - organization=params.get('organization', u''), - home_page=params.get('home_page', u''), - twitter=params.get('twitter', u''), - metadata=metadata + city=params.get("city", ""), + country=params.get("country", ""), + organization=params.get("organization", ""), + home_page=params.get("home_page", ""), + twitter=params.get("twitter", ""), + metadata=metadata, ) profile.save() return profile @@ -284,28 +299,28 @@ def validate_username(self, value): """ Validate username. """ - username = value.lower() if isinstance(value, basestring) else value + username = value.lower() if isinstance(value, str) else value if username in RESERVED_NAMES: - raise serializers.ValidationError(_( - u"%s is a reserved name, please choose another" % username - )) + raise serializers.ValidationError( + _("%s is a reserved name, please choose another" % username) + ) elif not LEGAL_USERNAMES_REGEX.search(username): - raise serializers.ValidationError(_( - u"username may only contain alpha-numeric characters and " - u"underscores" - )) + raise serializers.ValidationError( + _( + "username may only contain alpha-numeric characters and " + "underscores" + ) + ) elif len(username) < 3: - raise serializers.ValidationError(_( - u"Username must have 3 or more characters" - )) + raise serializers.ValidationError( + _("Username must have 3 or more characters") + ) users = User.objects.filter(username=username) if self.instance: users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"%s already exists" % username - )) + raise serializers.ValidationError(_("%s already exists" % username)) return username @@ -318,9 +333,9 @@ def validate_email(self, value): users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_( - u"This email address is already in use. " - )) + raise serializers.ValidationError( + _("This email address is already in use. ") + ) return value @@ -328,22 +343,25 @@ def validate_twitter(self, value): # pylint: disable=no-self-use """ Checks if the twitter handle is valid. """ - if isinstance(value, basestring) and value: + if isinstance(value, str) and value: match = re.search(r"^[A-Za-z0-9_]{1,15}$", value) if not match: - raise serializers.ValidationError(_( - u"Invalid twitter username {}".format(value) - )) + raise serializers.ValidationError( + _("Invalid twitter username {}".format(value)) + ) return value def validate(self, attrs): params = _get_registration_params(attrs) - if not self.instance and params.get('name') is None and \ - params.get('first_name') is None: - raise serializers.ValidationError({ - 'name': _(u"Either name or first_name should be provided") - }) + if ( + not self.instance + and params.get("name") is None + and params.get("first_name") is None + ): + raise serializers.ValidationError( + {"name": _("Either name or first_name should be provided")} + ) return params @@ -352,23 +370,38 @@ class UserProfileWithTokenSerializer(serializers.HyperlinkedModelSerializer): """ User Profile Serializer that includes the users API Tokens. """ + url = serializers.HyperlinkedIdentityField( - view_name='userprofile-detail', - lookup_field='user') - username = serializers.CharField(source='user.username') - email = serializers.CharField(source='user.email') - website = serializers.CharField(source='home_page', required=False) + view_name="userprofile-detail", lookup_field="user" + ) + username = serializers.CharField(source="user.username") + email = serializers.CharField(source="user.email") + website = serializers.CharField(source="home_page", required=False) gravatar = serializers.ReadOnlyField() user = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) api_token = serializers.SerializerMethodField() temp_token = serializers.SerializerMethodField() class Meta: model = UserProfile - fields = ('url', 'username', 'name', 'email', 'city', - 'country', 'organization', 'website', 'twitter', 'gravatar', - 'require_auth', 'user', 'api_token', 'temp_token') + fields = ( + "url", + "username", + "name", + "email", + "city", + "country", + "organization", + "website", + "twitter", + "gravatar", + "require_auth", + "user", + "api_token", + "temp_token", + ) def get_api_token(self, object): # pylint: disable=R0201,W0622 """ @@ -381,7 +414,7 @@ def get_temp_token(self, object): # pylint: disable=R0201,W0622 This should return a valid temp token for this user profile. """ token, created = TempToken.objects.get_or_create(user=object.user) - check_expired = getattr(settings, 'CHECK_EXPIRED_TEMP_TOKEN', True) + check_expired = getattr(settings, "CHECK_EXPIRED_TEMP_TOKEN", True) try: if check_expired and not created and expired(token.created): diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index e5c4f8677f..bd403ada5c 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -18,285 +18,288 @@ from openpyxl import load_workbook from django.conf import settings from django.core.files.temp import NamedTemporaryFile -from past.builtins import basestring +from openpyxl import load_workbook from pyxform.builder import create_survey_from_xls from savReaderWriter import SavHeaderReader, SavReader from onadata.apps.logger.import_tools import django_file from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (_encode_for_mongo, - query_data) +from onadata.apps.viewer.models.parsed_instance import _encode_for_mongo, query_data from onadata.apps.viewer.tests.export_helpers import viewer_fixture_path -from onadata.libs.utils.csv_builder import (CSVDataFrameBuilder, - get_labels_from_columns) +from onadata.libs.utils.csv_builder import CSVDataFrameBuilder, get_labels_from_columns from onadata.libs.utils.common_tags import ( - SELECT_BIND_TYPE, MULTIPLE_SELECT_TYPE, REVIEW_COMMENT, REVIEW_DATE, - REVIEW_STATUS) + SELECT_BIND_TYPE, + MULTIPLE_SELECT_TYPE, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, +) from onadata.libs.utils.export_builder import ( decode_mongo_encoded_section_names, dict_to_joined_export, ExportBuilder, - string_to_date_with_xls_validation) + string_to_date_with_xls_validation, +) from onadata.libs.utils.export_tools import get_columns_with_hxl from onadata.libs.utils.logger_tools import create_instance def _logger_fixture_path(*args): - return os.path.join(settings.PROJECT_ROOT, 'apps', 'logger', - 'tests', 'fixtures', *args) + return os.path.join( + settings.PROJECT_ROOT, "apps", "logger", "tests", "fixtures", *args + ) class TestExportBuilder(TestBase): data = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', - 'children': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", + "children": [ { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/iceLg==creams': 'vanilla chocolate', - 'children/cartoons': - [ + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/iceLg==creams": "vanilla chocolate", + "children/cartoons": [ { - 'children/cartoons/name': 'Tom & Jerry', - 'children/cartoons/why': 'Tom is silly', + "children/cartoons/name": "Tom & Jerry", + "children/cartoons/why": "Tom is silly", }, { - 'children/cartoons/name': 'Flinstones', - 'children/cartoons/why': u"I like bam bam\u0107" + "children/cartoons/name": "Flinstones", + "children/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] - }, - { - 'children/name': 'John', - 'children/age': 2, - 'children/cartoons': [] + }, + ], }, + {"children/name": "John", "children/age": 2, "children/cartoons": []}, { - 'children/name': 'Imora', - 'children/age': 3, - 'children/cartoons': - [ + "children/name": "Imora", + "children/age": 3, + "children/cartoons": [ { - 'children/cartoons/name': 'Shrek', - 'children/cartoons/why': 'He\'s so funny' + "children/cartoons/name": "Shrek", + "children/cartoons/why": "He's so funny", }, { - 'children/cartoons/name': 'Dexter\'s Lab', - 'children/cartoons/why': 'He thinks hes smart', - 'children/cartoons/characters': - [ + "children/cartoons/name": "Dexter's Lab", + "children/cartoons/why": "He thinks hes smart", + "children/cartoons/characters": [ { - 'children/cartoons/characters/name': - 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "children/cartoons/characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'children/cartoons/characters/name': - 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "children/cartoons/characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], }, { # blank data just to be sure - 'children': [] - } + "children": [] + }, ] long_survey_data = [ { - 'name': 'Abe', - 'age': 35, - 'childrens_survey_with_a_very_lo': - [ + "name": "Abe", + "age": 35, + "childrens_survey_with_a_very_lo": [ { - 'childrens_survey_with_a_very_lo/name': 'Mike', - 'childrens_survey_with_a_very_lo/age': 5, - 'childrens_survey_with_a_very_lo/fav_colors': 'red blue', - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Mike", + "childrens_survey_with_a_very_lo/age": 5, + "childrens_survey_with_a_very_lo/fav_colors": "red blue", + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Tom & Jerry', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'Tom is silly', + "childrens_survey_with_a_very_lo/cartoons/name": "Tom & Jerry", + "childrens_survey_with_a_very_lo/cartoons/why": "Tom is silly", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Flinstones', - 'childrens_survey_with_a_very_lo/cartoons/why': - u"I like bam bam\u0107" + "childrens_survey_with_a_very_lo/cartoons/name": "Flinstones", + "childrens_survey_with_a_very_lo/cartoons/why": "I like bam bam\u0107" # throw in a unicode character - } - ] + }, + ], }, { - 'childrens_survey_with_a_very_lo/name': 'John', - 'childrens_survey_with_a_very_lo/age': 2, - 'childrens_survey_with_a_very_lo/cartoons': [] + "childrens_survey_with_a_very_lo/name": "John", + "childrens_survey_with_a_very_lo/age": 2, + "childrens_survey_with_a_very_lo/cartoons": [], }, { - 'childrens_survey_with_a_very_lo/name': 'Imora', - 'childrens_survey_with_a_very_lo/age': 3, - 'childrens_survey_with_a_very_lo/cartoons': - [ + "childrens_survey_with_a_very_lo/name": "Imora", + "childrens_survey_with_a_very_lo/age": 3, + "childrens_survey_with_a_very_lo/cartoons": [ { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Shrek', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He\'s so funny' + "childrens_survey_with_a_very_lo/cartoons/name": "Shrek", + "childrens_survey_with_a_very_lo/cartoons/why": "He's so funny", }, { - 'childrens_survey_with_a_very_lo/cartoons/name': - 'Dexter\'s Lab', - 'childrens_survey_with_a_very_lo/cartoons/why': - 'He thinks hes smart', - 'childrens_survey_with_a_very_lo/cartoons/characte' - 'rs': - [ + "childrens_survey_with_a_very_lo/cartoons/name": "Dexter's Lab", + "childrens_survey_with_a_very_lo/cartoons/why": "He thinks hes smart", + "childrens_survey_with_a_very_lo/cartoons/characte" + "rs": [ { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dee Dee', - 'children/cartoons/characters/good_or_evi' - 'l': 'good' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dee Dee", + "children/cartoons/characters/good_or_evi" + "l": "good", }, { - 'childrens_survey_with_a_very_lo/cartoons/' - 'characters/name': 'Dexter', - 'children/cartoons/characters/good_or_evi' - 'l': 'evil' + "childrens_survey_with_a_very_lo/cartoons/" + "characters/name": "Dexter", + "children/cartoons/characters/good_or_evi" + "l": "evil", }, - ] - } - ] - } - ] + ], + }, + ], + }, + ], } ] data_utf8 = [ { - 'name': 'Abe', - 'age': 35, - 'tel/telLg==office': '020123456', - 'childrenLg==info': - [ + "name": "Abe", + "age": 35, + "tel/telLg==office": "020123456", + "childrenLg==info": [ { - 'childrenLg==info/nameLg==first': 'Mike', - 'childrenLg==info/age': 5, - 'childrenLg==info/fav_colors': "red's blue's", - 'childrenLg==info/ice_creams': 'vanilla chocolate', - 'childrenLg==info/cartoons': - [ + "childrenLg==info/nameLg==first": "Mike", + "childrenLg==info/age": 5, + "childrenLg==info/fav_colors": "red's blue's", + "childrenLg==info/ice_creams": "vanilla chocolate", + "childrenLg==info/cartoons": [ { - 'childrenLg==info/cartoons/name': 'Tom & Jerry', - 'childrenLg==info/cartoons/why': 'Tom is silly', + "childrenLg==info/cartoons/name": "Tom & Jerry", + "childrenLg==info/cartoons/why": "Tom is silly", }, { - 'childrenLg==info/cartoons/name': 'Flinstones', - 'childrenLg==info/cartoons/why': + "childrenLg==info/cartoons/name": "Flinstones", + "childrenLg==info/cartoons/why": # throw in a unicode character - 'I like bam bam\u0107' - } - ] + "I like bam bam\u0107", + }, + ], } - ] + ], } ] osm_data = [ { - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road:ctr:lat': '23.708174238006087', - 'osm_road:ctr:lon': '90.40946505581161', - 'osm_road:highway': 'tertiary', - 'osm_road:lanes': 2, - 'osm_road:name': 'Patuatuli Road', - 'osm_building:building': 'yes', - 'osm_building:building:levels': 4, - 'osm_building:ctr:lat': '23.707316084046038', - 'osm_building:ctr:lon': '90.40849938337506', - 'osm_building:name': 'kol', - '_review_status': 'Rejected', - '_review_comment': 'Wrong Location', - REVIEW_DATE: '2021-05-25T02:27:19', + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road:ctr:lat": "23.708174238006087", + "osm_road:ctr:lon": "90.40946505581161", + "osm_road:highway": "tertiary", + "osm_road:lanes": 2, + "osm_road:name": "Patuatuli Road", + "osm_building:building": "yes", + "osm_building:building:levels": 4, + "osm_building:ctr:lat": "23.707316084046038", + "osm_building:ctr:lon": "90.40849938337506", + "osm_building:name": "kol", + "_review_status": "Rejected", + "_review_comment": "Wrong Location", + REVIEW_DATE: "2021-05-25T02:27:19", } ] def _create_childrens_survey(self, filename="childrens_survey.xlsx"): - survey = create_survey_from_xls(_logger_fixture_path( - filename - ), default_name=filename.split('.')[0]) + survey = create_survey_from_xls( + _logger_fixture_path(filename), default_name=filename.split(".")[0] + ) self.dd = DataDictionary() self.dd._survey = survey return survey def test_build_sections_for_multilanguage_form(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'multi_lingual_form.xlsx'), - default_name='multi_lingual_form') + survey = create_survey_from_xls( + _logger_fixture_path("multi_lingual_form.xlsx"), + default_name="multi_lingual_form", + ) # check the default langauge - self.assertEqual( - survey.to_json_dict().get('default_language'), 'English' - ) + self.assertEqual(survey.to_json_dict().get("default_language"), "English") export_builder = ExportBuilder() export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - expected_sections = [ - survey.name] + expected_sections = [survey.name] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_sections, [s["name"] for s in export_builder.sections] + ) + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - expected_element_names = \ - ['Des fruits', 'Fruits/Aucune de ces réponses', 'Fruits/Autre', - 'Fruits/Banane', 'Fruits/Mangue', 'Fruits/Poire', 'Fruits/Pomme', - "L'age", 'Le genre', 'Nom de personne interrogée', 'Villes', - 'meta/instanceID'] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + expected_element_names = [ + "Des fruits", + "Fruits/Aucune de ces réponses", + "Fruits/Autre", + "Fruits/Banane", + "Fruits/Mangue", + "Fruits/Poire", + "Fruits/Pomme", + "L'age", + "Le genre", + "Nom de personne interrogée", + "Villes", + "meta/instanceID", + ] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) # use default language when the language passed does not exist - export_builder.language = 'Kiswahili' + export_builder.language = "Kiswahili" export_builder.set_survey(survey) - expected_element_names = \ - ['Name of respondent', 'Age', 'Sex of respondent', 'Fruits', - 'Fruits/Apple', 'Fruits/Banana', 'Fruits/Pear', 'Fruits/Mango', - 'Fruits/Other', 'Fruits/None of the above', 'Cities', - 'meta/instanceID'] + expected_element_names = [ + "Name of respondent", + "Age", + "Sex of respondent", + "Fruits", + "Fruits/Apple", + "Fruits/Banana", + "Fruits/Pear", + "Fruits/Mango", + "Fruits/Other", + "Fruits/None of the above", + "Cities", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['label'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + element_names = [element["label"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) def test_build_sections_from_survey(self): survey = self._create_childrens_survey() @@ -304,59 +307,71 @@ def test_build_sections_from_survey(self): export_builder.set_survey(survey) # test that we generate the proper sections expected_sections = [ - survey.name, 'children', 'children/cartoons', - 'children/cartoons/characters'] + survey.name, + "children", + "children/cartoons", + "children/cartoons/characters", + ] self.assertEqual( - expected_sections, [s['name'] for s in export_builder.sections]) + expected_sections, [s["name"] for s in export_builder.sections] + ) # main section should have split geolocations expected_element_names = [ - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_longitude', - 'geo/_geolocation_latitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', 'tel/tel.mobile', - 'meta/instanceID'] + "name", + "age", + "geo/geolocation", + "geo/_geolocation_longitude", + "geo/_geolocation_latitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "meta/instanceID", + ] section = export_builder.section_by_name(survey.name) - element_names = [element['xpath'] for element in section['elements']] + element_names = [element["xpath"] for element in section["elements"]] # fav_colors should have its choices split - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate'] - section = export_builder.section_by_name('children') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ] + section = export_builder.section_by_name("children") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) - expected_element_names = [ - 'children/cartoons/name', 'children/cartoons/why'] - section = export_builder.section_by_name('children/cartoons') - element_names = [element['xpath'] for element in section['elements']] + expected_element_names = ["children/cartoons/name", "children/cartoons/why"] + section = export_builder.section_by_name("children/cartoons") + element_names = [element["xpath"] for element in section["elements"]] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil'] - section = \ - export_builder.section_by_name('children/cartoons/characters') - element_names = [element['xpath'] for element in section['elements']] - self.assertEqual( - sorted(expected_element_names), sorted(element_names)) + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + ] + section = export_builder.section_by_name("children/cartoons/characters") + element_names = [element["xpath"] for element in section["elements"]] + self.assertEqual(sorted(expected_element_names), sorted(element_names)) def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() @@ -368,68 +383,70 @@ def test_zipped_csv_export_works(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.csv".format(survey.name)))) - with open(os.path.join(temp_dir, "{0}.csv".format(survey.name)), - encoding='utf-8') as csv_file: + os.path.exists(os.path.join(temp_dir, "{0}.csv".format(survey.name))) + ) + with open( + os.path.join(temp_dir, "{0}.csv".format(survey.name)), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'childrens_survey.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "childrens_survey.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.csv"))) - with open(os.path.join(temp_dir, "children.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) + with open(os.path.join(temp_dir, "children.csv"), encoding="utf-8") as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'children.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons.csv"))) - with open(os.path.join(temp_dir, "children_cartoons.csv"), - encoding='utf-8') as csv_file: + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) + with open( + os.path.join(temp_dir, "children_cartoons.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path('csvs', 'children_cartoons.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children_cartoons.csv"), encoding="utf-8" + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children_cartoons_characters.csv"))) - with open(os.path.join(temp_dir, "children_cartoons_characters.csv"), - encoding='utf-8') as csv_file: + os.path.exists(os.path.join(temp_dir, "children_cartoons_characters.csv")) + ) + with open( + os.path.join(temp_dir, "children_cartoons_characters.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file - with open(_logger_fixture_path( - 'csvs', 'children_cartoons_characters.csv'), - encoding='utf-8') as fixture_csv: + with open( + _logger_fixture_path("csvs", "children_cartoons_characters.csv"), + encoding="utf-8", + ) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) @@ -444,7 +461,7 @@ def test_xls_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.osm_data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) @@ -454,37 +471,57 @@ def test_xls_export_with_osm_data(self): temp_xls_file.close() expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(expected_column_headers), sorted(xls_headers)) submission = [a.value for a in rows[1]] - self.assertEqual(submission[0], '1424308569120.jpg') - self.assertEqual(submission[2], '23.708174238006087') - self.assertEqual(submission[4], 'tertiary') - self.assertEqual(submission[6], 'Patuatuli Road') - self.assertEqual(submission[11], '23.707316084046038') - self.assertEqual(submission[13], 'kol') + self.assertEqual(submission[0], "1424308569120.jpg") + self.assertEqual(submission[2], "23.708174238006087") + self.assertEqual(submission[4], "tertiary") + self.assertEqual(submission[6], "Patuatuli Road") + self.assertEqual(submission[11], "23.707316084046038") + self.assertEqual(submission[13], "kol") def test_decode_mongo_encoded_section_names(self): data = { - 'main_section': [1, 2, 3, 4], - 'sectionLg==1/info': [1, 2, 3, 4], - 'sectionLg==2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "sectionLg==1/info": [1, 2, 3, 4], + "sectionLg==2/info": [1, 2, 3, 4], } result = decode_mongo_encoded_section_names(data) expected_result = { - 'main_section': [1, 2, 3, 4], - 'section.1/info': [1, 2, 3, 4], - 'section.2/info': [1, 2, 3, 4], + "main_section": [1, 2, 3, 4], + "section.1/info": [1, 2, 3, 4], + "section.2/info": [1, 2, 3, 4], } self.assertEqual(result, expected_result) @@ -492,12 +529,13 @@ def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -506,34 +544,42 @@ def test_zipped_csv_export_works_with_unicode(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['children.info/name.first', - 'children.info/age', - 'children.info/fav_colors', - 'children.info/fav_colors/red\'s', - 'children.info/fav_colors/blue\'s', - 'children.info/fav_colors/pink\'s', - 'children.info/ice_creams', - 'children.info/ice_creams/vanilla', - 'children.info/ice_creams/strawberry', - 'children.info/ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "children.info/name.first", + "children.info/age", + "children.info/fav_colors", + "children.info/fav_colors/red's", + "children.info/fav_colors/blue's", + "children.info/fav_colors/pink's", + "children.info/ice_creams", + "children.info/ice_creams/vanilla", + "children.info/ice_creams/strawberry", + "children.info/ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['children.info/fav_colors/red\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/blue\'s'], 'True') - self.assertEqual(data['children.info/fav_colors/pink\'s'], 'False') + self.assertEqual(data["children.info/fav_colors/red's"], "True") + self.assertEqual(data["children.info/fav_colors/blue's"], "True") + self.assertEqual(data["children.info/fav_colors/pink's"], "False") # check that red and blue are set to true def test_zipped_sav_export_with_date_field(self): @@ -548,12 +594,17 @@ def test_zipped_sav_export_with_date_field(self): | choices | | | list name | name | label | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expense_date": "2013-01-03", "A/gdate": "2017-06-13", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expense_date": "2013-01-03", + "A/gdate": "2017-06-13", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -562,21 +613,18 @@ def test_zipped_sav_export_with_date_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'expense_date') - self.assertEqual(rows[1][0], b'2013-01-03') - self.assertEqual(rows[0][1], b'A.gdate') - self.assertEqual(rows[1][1], b'2017-06-13') - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][0], b"expense_date") + self.assertEqual(rows[1][0], b"2013-01-03") + self.assertEqual(rows[0][1], b"A.gdate") + self.assertEqual(rows[1][1], b"2017-06-13") + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -599,13 +647,19 @@ def test_zipped_sav_export_dynamic_select_multiple(self): | | brand | a | a | | | brand | b | b | """ # noqa - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) data = [ - {"sex": "male", "text": "his", "favorite_brand": "Generic", - "name": "Davis", "brand_known": "${text} ${favorite_brand} a"}] + { + "sex": "male", + "text": "his", + "favorite_brand": "Generic", + "name": "Davis", + "brand_known": "${text} ${favorite_brand} a", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -614,31 +668,28 @@ def test_zipped_sav_export_dynamic_select_multiple(self): zip_file.close() temp_zip_file.close() - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b'sex') - self.assertEqual(rows[1][0], b'male') - self.assertEqual(rows[0][1], b'text') - self.assertEqual(rows[1][1], b'his') - self.assertEqual(rows[0][2], b'favorite_brand') - self.assertEqual(rows[1][2], b'Generic') - self.assertEqual(rows[0][3], b'name') - self.assertEqual(rows[1][3], b'Davis') - self.assertEqual(rows[0][4], b'brand_known') - self.assertEqual(rows[1][4], b'his Generic a') - self.assertEqual(rows[0][5], b'brand_known.$text') + self.assertEqual(rows[0][0], b"sex") + self.assertEqual(rows[1][0], b"male") + self.assertEqual(rows[0][1], b"text") + self.assertEqual(rows[1][1], b"his") + self.assertEqual(rows[0][2], b"favorite_brand") + self.assertEqual(rows[1][2], b"Generic") + self.assertEqual(rows[0][3], b"name") + self.assertEqual(rows[1][3], b"Davis") + self.assertEqual(rows[0][4], b"brand_known") + self.assertEqual(rows[1][4], b"his Generic a") + self.assertEqual(rows[0][5], b"brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], b'brand_known.$favorite_brand') + self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], b'brand_known.a') + self.assertEqual(rows[0][7], b"brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], b'brand_known.b') + self.assertEqual(rows[0][8], b"brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) @@ -654,31 +705,27 @@ def test_zipped_sav_export_with_zero_padded_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{'expensed': '09', - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "09", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, 'r') + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, 'exp.sav'))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, 'exp.sav'), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0].decode('utf-8'), '09') - self.assertEqual(rows[1][4].decode('utf-8'), '2016-11-21 03:43:43') + self.assertEqual(rows[1][0].decode("utf-8"), "09") + self.assertEqual(rows[1][4].decode("utf-8"), "2016-11-21 03:43:43") def test_zipped_sav_export_with_numeric_select_one_field(self): md = """ @@ -694,12 +741,17 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): | | yes_no | 1 | Yes | | | yes_no | 0 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -708,27 +760,24 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], b'expensed') + self.assertEqual(rows[0][0], b"expensed") self.assertEqual(rows[1][0], 1) # A/q1 1 - self.assertEqual(rows[0][1], b'A.q1') + self.assertEqual(rows[0][1], b"A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], b'@_submission_time') - self.assertEqual(rows[1][5], b'2016-11-21 03:43:43') + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") def test_zipped_sav_export_with_duplicate_field_different_groups(self): """ @@ -759,20 +808,20 @@ def test_zipped_sav_export_with_duplicate_field_different_groups(self): | | x_y | 2 | Non | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - labels = export_builder._get_sav_value_labels({'A/allaite': 'allaite'}) - self.assertEqual(labels, {'allaite': {'1': 'Oui', '2': 'Non'}}) + labels = export_builder._get_sav_value_labels({"A/allaite": "allaite"}) + self.assertEqual(labels, {"allaite": {"1": "Oui", "2": "Non"}}) repeat_group_labels = export_builder._get_sav_value_labels( - {'A/rep/allaite': 'allaite'}) - self.assertEqual(repeat_group_labels, - {'allaite': {1: 'Yes', 2: 'No'}}) + {"A/rep/allaite": "allaite"} + ) + self.assertEqual(repeat_group_labels, {"allaite": {1: "Yes", 2: "No"}}) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") try: export_builder.to_zipped_sav(temp_zip_file.name, []) @@ -812,54 +861,55 @@ def test_split_select_multiples_choices_with_randomize_param(self): | | allow_choice_duplicates | | | Yes | """ # noqa: E501 - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) @@ -879,12 +929,18 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): | | y_n | 0 | No | | | | | | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", "A/q1": "1", "A/q2": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + { + "expensed": "1", + "A/q1": "1", + "A/q2": "1", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -893,13 +949,10 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) @@ -931,7 +984,7 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): self.assertEqual(rows[1][5], 0) self.assertEqual(rows[0][12], b"@_submission_time") - self.assertEqual(rows[1][12], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -946,12 +999,11 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): | | yes_no | 1 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [{"expensed": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -960,13 +1012,10 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"1") @@ -974,7 +1023,7 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -989,13 +1038,14 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): | | yes_no | 2 | Yes | | | yes_no | 09 | No | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"expensed": "2 09", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"expensed": "2 09", "_submission_time": "2016-11-21T03:43:43.000-08:00"} + ] export_builder = ExportBuilder() export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1004,21 +1054,18 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "exp.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = [r for r in reader] self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 2) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[1][2], b'09') - self.assertEqual(rows[1][6], b'2016-11-21 03:43:43') + self.assertEqual(rows[1][2], b"09") + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1044,15 +1091,14 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): | | allow_choice_duplicates | | | Yes | """ - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'} - ] + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1061,9 +1107,7 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 """ @@ -1074,14 +1118,14 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 | | type | name | label | | | select_one_from_file animals.csv | q1 | Favorite animal? | """ - survey = self.md_to_pyxform_survey(xform_markdown, {'name': 'exp'}) - data = [{"q1": "1", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}, - {"q1": "6", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(xform_markdown, {"name": "exp"}) + data = [ + {"q1": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + {"q1": "6", "_submission_time": "2016-11-21T03:43:43.000-08:00"}, + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1090,9 +1134,7 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "exp.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) def test_zipped_sav_export_with_duplicate_column_name(self): """ @@ -1104,13 +1146,18 @@ def test_zipped_sav_export_with_duplicate_column_name(self): | | text | Sport | Which sport | | | text | sport | Which fun sports| """ - survey = self.md_to_pyxform_survey(md, {'name': 'sports'}) - data = [{"Sport": "Basketball", "sport": "Soccer", - '_submission_time': '2016-11-21T03:43:43.000-08:00'}] + survey = self.md_to_pyxform_survey(md, {"name": "sports"}) + data = [ + { + "Sport": "Basketball", + "sport": "Soccer", + "_submission_time": "2016-11-21T03:43:43.000-08:00", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -1119,13 +1166,12 @@ def test_zipped_sav_export_with_duplicate_column_name(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "sports.sav"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "sports.sav"))) # check file's contents - with SavReader(os.path.join(temp_dir, "sports.sav"), - returnHeader=True) as reader: + with SavReader( + os.path.join(temp_dir, "sports.sav"), returnHeader=True + ) as reader: rows = [r for r in reader] # Check that columns are present @@ -1135,12 +1181,13 @@ def test_zipped_sav_export_with_duplicate_column_name(self): self.assertIn(b"sport", [x[0:5] for x in rows[0]]) def test_xls_export_works_with_unicode(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrenss_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrenss_survey_unicode", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true @@ -1155,20 +1202,26 @@ def test_xls_export_works_with_unicode(self): def test_xls_export_with_hxl_adds_extra_row(self): # hxl_example.xlsx contains `instance::hxl` column whose value is #age xlsform_path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "hxl_test", "hxl_example.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "hxl_test", + "hxl_example.xlsx", + ) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) export_builder = ExportBuilder() export_builder.INCLUDE_HXL = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") survey_elements = [ survey_item[1] for survey_item in survey.items() - if survey_item[0] == 'children' + if survey_item[0] == "children" ][0] columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( @@ -1176,8 +1229,8 @@ def test_xls_export_with_hxl_adds_extra_row(self): ) export_builder.to_xls_export( - temp_xls_file.name, self.data_utf8, - columns_with_hxl=columns_with_hxl) + temp_xls_file.name, self.data_utf8, columns_with_hxl=columns_with_hxl + ) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) children_sheet = wb["hxl_example"] @@ -1186,7 +1239,7 @@ def test_xls_export_with_hxl_adds_extra_row(self): # we pick the second row because the first row has xform fieldnames rows = [row for row in children_sheet.rows] hxl_row = [a.value for a in rows[1]] - self.assertIn('#age', hxl_row) + self.assertIn("#age", hxl_row) def test_export_with_image_attachments(self): """ @@ -1207,23 +1260,29 @@ def test_export_with_image_attachments(self): 1300221157303.jpg - """.format(self.xform.id_string) - - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - media_file = django_file(path=file_path, - field_name="image1", - content_type="image/jpeg") - create_instance(self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + """.format( + self.xform.id_string + ) + + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + media_file = django_file( + path=file_path, field_name="image1", content_type="image/jpeg" + ) + create_instance( + self.user.username, + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) xdata = query_data(self.xform) - survey = self.md_to_pyxform_survey(md, {'name': 'exp'}) + survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file, xdata) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file) @@ -1231,9 +1290,11 @@ def test_export_with_image_attachments(self): self.assertTrue(children_sheet) rows = [row for row in children_sheet.rows] row = [a.value for a in rows[1]] - attachment_id = xdata[0]['_attachments'][0]['id'] - attachment_filename = xdata[0]['_attachments'][0]['filename'] - attachment_url = 'http://example.com/api/v1/files/{}?filename={}'.format(attachment_id, attachment_filename) # noqa + attachment_id = xdata[0]["_attachments"][0]["id"] + attachment_filename = xdata[0]["_attachments"][0]["filename"] + attachment_url = "http://example.com/api/v1/files/{}?filename={}".format( + attachment_id, attachment_filename + ) # noqa self.assertIn(attachment_url, row) temp_xls_file.close() @@ -1241,112 +1302,113 @@ def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_select_multiples =\ - { - 'children': - { - 'children/fav_colors': - [ - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink' - ], - 'children/ice.creams': - [ - 'children/ice.creams/vanilla', - 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate' - ] - } + expected_select_multiples = { + "children": { + "children/fav_colors": [ + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + ], + "children/ice.creams": [ + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + ], } + } select_multiples = export_builder.select_multiples - self.assertTrue('children' in select_multiples) - self.assertTrue('children/fav_colors' in select_multiples['children']) - self.assertTrue('children/ice.creams' in select_multiples['children']) + self.assertTrue("children" in select_multiples) + self.assertTrue("children/fav_colors" in select_multiples["children"]) + self.assertTrue("children/ice.creams" in select_multiples["children"]) self.assertEqual( - sorted([ - choice['xpath'] for choice in - select_multiples['children']['children/fav_colors']]), sorted( - expected_select_multiples['children']['children/fav_colors'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/fav_colors"] + ] + ), + sorted(expected_select_multiples["children"]["children/fav_colors"]), + ) self.assertEqual( - sorted([choice['xpath'] for choice in - select_multiples['children']['children/ice.creams']]), sorted( - expected_select_multiples['children']['children/ice.creams'])) + [ + choice["xpath"] + for choice in select_multiples["children"]["children/ice.creams"] + ] + ), + sorted(expected_select_multiples["children"]["children/ice.creams"]), + ) def test_split_select_multiples_works(self): """ Test split_select_multiples works as expected. """ - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': 'red blue', - 'children/fav_colors/red': True, - 'children/fav_colors/blue': True, - 'children/fav_colors/pink': False - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "red blue", + "children/fav_colors/red": True, + "children/fav_colors/blue": True, + "children/fav_colors/pink": False, + } self.assertEqual(new_row, expected_row) - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + row = { + "children/name": "Mike", + "children/age": 5, + } + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) def test_split_select_mutliples_works_with_int_value_in_row(self): select_multiples = { - 'children/fav_number': [ + "children/fav_number": [ { - 'xpath': 'children/fav_number/1', - }, { - 'xpath': 'children/fav_number/2', - }, { - 'xpath': 'children/fav_number/3', - } + "xpath": "children/fav_number/1", + }, + { + "xpath": "children/fav_number/2", + }, + { + "xpath": "children/fav_number/3", + }, ] } - row = {'children/fav_number': 1} + row = {"children/fav_number": 1} expected_row = { - 'children/fav_number/1': True, - 'children/fav_number': 1, - 'children/fav_number/3': False, - 'children/fav_number/2': False + "children/fav_number/1": True, + "children/fav_number": 1, + "children/fav_number/3": False, + "children/fav_number/2": False, } new_row = ExportBuilder.split_select_multiples(row, select_multiples) @@ -1354,156 +1416,130 @@ def test_split_select_mutliples_works_with_int_value_in_row(self): self.assertEqual(new_row, expected_row) def test_split_select_multiples_works_when_data_is_blank(self): - select_multiples =\ - { - 'children/fav_colors': [ - { - 'xpath': 'children/fav_colors/red', - 'label': 'fav_colors/Red', - }, { - 'xpath': 'children/fav_colors/blue', - 'label': 'fav_colors/Blue', - }, { - 'xpath': 'children/fav_colors/pink', - 'label': 'fav_colors/Pink', - } - ] - } - row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '' - } - new_row = ExportBuilder.split_select_multiples( - row, select_multiples) - expected_row = \ - { - 'children/name': 'Mike', - 'children/age': 5, - 'children/fav_colors': '', - 'children/fav_colors/red': None, - 'children/fav_colors/blue': None, - 'children/fav_colors/pink': None - } + select_multiples = { + "children/fav_colors": [ + { + "xpath": "children/fav_colors/red", + "label": "fav_colors/Red", + }, + { + "xpath": "children/fav_colors/blue", + "label": "fav_colors/Blue", + }, + { + "xpath": "children/fav_colors/pink", + "label": "fav_colors/Pink", + }, + ] + } + row = {"children/name": "Mike", "children/age": 5, "children/fav_colors": ""} + new_row = ExportBuilder.split_select_multiples(row, select_multiples) + expected_row = { + "children/name": "Mike", + "children/age": 5, + "children/fav_colors": "", + "children/fav_colors/red": None, + "children/fav_colors/blue": None, + "children/fav_colors/pink": None, + } self.assertEqual(new_row, expected_row) def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_gps_fields =\ - { - 'childrens_survey': - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', - 'geo/_geolocation_precision' - ] - } + expected_gps_fields = { + "childrens_survey": { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] } + } gps_fields = export_builder.gps_fields - self.assertTrue('childrens_survey' in gps_fields) + self.assertTrue("childrens_survey" in gps_fields) self.assertEqual( - sorted(gps_fields['childrens_survey']), - sorted(expected_gps_fields['childrens_survey'])) + sorted(gps_fields["childrens_survey"]), + sorted(expected_gps_fields["childrens_survey"]), + ) def test_split_gps_components_works(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '1.0 36.1 2000 20', - 'geo/_geolocation_latitude': '1.0', - 'geo/_geolocation_longitude': '36.1', - 'geo/_geolocation_altitude': '2000', - 'geo/_geolocation_precision': '20' - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "1.0 36.1 2000 20", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "1.0 36.1 2000 20", + "geo/_geolocation_latitude": "1.0", + "geo/_geolocation_longitude": "36.1", + "geo/_geolocation_altitude": "2000", + "geo/_geolocation_precision": "20", + } self.assertEqual(new_row, expected_row) def test_split_gps_components_works_when_gps_data_is_blank(self): - gps_fields =\ - { - 'geo/geolocation': - [ - 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', - 'geo/_geolocation_altitude', 'geo/_geolocation_precision' - ] - } - row = \ - { - 'geo/geolocation': '', - } - new_row = ExportBuilder.split_gps_components( - row, gps_fields) - expected_row = \ - { - 'geo/geolocation': '', - } + gps_fields = { + "geo/geolocation": [ + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + ] + } + row = { + "geo/geolocation": "", + } + new_row = ExportBuilder.split_gps_components(row, gps_fields) + expected_row = { + "geo/geolocation": "", + } self.assertEqual(new_row, expected_row) def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - expected_encoded_fields =\ - { - 'childrens_survey': - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')), - 'tel/tel.mobile': 'tel/{0}'.format( - _encode_for_mongo('tel.mobile')), - } + expected_encoded_fields = { + "childrens_survey": { + "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")), + "tel/tel.mobile": "tel/{0}".format(_encode_for_mongo("tel.mobile")), } + } encoded_fields = export_builder.encoded_fields - self.assertTrue('childrens_survey' in encoded_fields) + self.assertTrue("childrens_survey" in encoded_fields) self.assertEqual( - encoded_fields['childrens_survey'], - expected_encoded_fields['childrens_survey']) + encoded_fields["childrens_survey"], + expected_encoded_fields["childrens_survey"], + ) def test_decode_fields_names_encoded_for_mongo(self): - encoded_fields = \ - { - 'tel/tel.office': 'tel/{0}'.format( - _encode_for_mongo('tel.office')) - } - row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/{0}'.format( - _encode_for_mongo('tel.office')): '123-456-789' - } - new_row = ExportBuilder.decode_mongo_encoded_fields( - row, encoded_fields) - expected_row = \ - { - 'name': 'Abe', - 'age': 35, - 'tel/tel.office': '123-456-789' - } + encoded_fields = { + "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")) + } + row = { + "name": "Abe", + "age": 35, + "tel/{0}".format(_encode_for_mongo("tel.office")): "123-456-789", + } + new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields) + expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"} self.assertEqual(new_row, expected_row) def test_generate_field_title(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", ".", - data_dictionary=self.dd) + field_name = ExportBuilder.format_field_title( + "children/age", ".", data_dictionary=self.dd + ) expected_field_name = "children.age" self.assertEqual(field_name, expected_field_name) @@ -1512,129 +1548,170 @@ def test_delimiter_replacement_works_existing_fields(self): export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_sections =\ - [ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.name', - 'xpath': 'children/name' - } - ] - } - ] - children_section = export_builder.section_by_name('children') + expected_sections = [ + { + "name": "children", + "elements": [{"title": "children.name", "xpath": "children/name"}], + } + ] + children_section = export_builder.section_by_name("children") self.assertEqual( - children_section['elements'][0]['title'], - expected_sections[0]['elements'][0]['title']) + children_section["elements"][0]["title"], + expected_sections[0]["elements"][0]["title"], + ) def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section =\ - { - 'name': 'children', - 'elements': [ - { - 'title': 'children.fav_colors.red', - 'xpath': 'children/fav_colors/red' - } - ] - } - childrens_section = export_builder.section_by_name('children') - match = [x for x in childrens_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "children", + "elements": [ + {"title": "children.fav_colors.red", "xpath": "children/fav_colors/red"} + ], + } + childrens_section = export_builder.section_by_name("children") + match = [ + x + for x in childrens_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) - expected_section = \ - { - 'name': 'childrens_survey', - 'elements': [ - { - 'title': 'geo._geolocation_latitude', - 'xpath': 'geo/_geolocation_latitude' - } - ] - } - main_section = export_builder.section_by_name('childrens_survey') - match = [x for x in main_section['elements'] - if expected_section['elements'][0]['xpath'] == x['xpath']][0] - self.assertEqual( - expected_section['elements'][0]['title'], match['title']) + expected_section = { + "name": "childrens_survey", + "elements": [ + { + "title": "geo._geolocation_latitude", + "xpath": "geo/_geolocation_latitude", + } + ], + } + main_section = export_builder.section_by_name("childrens_survey") + match = [ + x + for x in main_section["elements"] + if expected_section["elements"][0]["xpath"] == x["xpath"] + ][0] + self.assertEqual(expected_section["elements"][0]["title"], match["title"]) def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') + xls_file = NamedTemporaryFile(suffix=".xlsx") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey', 'children', - 'children_cartoons', - 'children_cartoons_characters'] - self.assertEqual(wb.get_sheet_names(), expected_sheet_names) + expected_sheet_names = [ + "childrens_survey", + "children", + "children_cartoons", + "children_cartoons_characters", + ] + self.assertEqual(wb.sheet_names(), expected_sheet_names) + # check header columns - main_sheet = wb.get_sheet_by_name('childrens_survey') - expected_column_headers = ( - 'name', 'age', 'geo/geolocation', 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', 'geo/_geolocation_altitude', - 'geo/_geolocation_precision', 'tel/tel.office', - 'tel/tel.mobile', '_id', 'meta/instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - - column_headers = tuple(main_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - childrens_sheet = wb.get_sheet_by_name('children') - expected_column_headers = ( - 'children/name', 'children/age', 'children/fav_colors', - 'children/fav_colors/red', 'children/fav_colors/blue', - 'children/fav_colors/pink', 'children/ice.creams', - 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', - 'children/ice.creams/chocolate', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(childrens_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - cartoons_sheet = wb.get_sheet_by_name('children_cartoons') - expected_column_headers = ( - 'children/cartoons/name', 'children/cartoons/why', '_id', - '_uuid', '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(cartoons_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) - - characters_sheet = wb.get_sheet_by_name('children_cartoons_characters') - expected_column_headers = ( - 'children/cartoons/characters/name', - 'children/cartoons/characters/good_or_evil', '_id', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(characters_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + main_sheet = wb.sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo/geolocation", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "_id", + "meta/instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = main_sheet.row_values(0) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + + childrens_sheet = wb.sheet_by_name("children") + expected_column_headers = [ + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = childrens_sheet.row_values(0) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + + cartoons_sheet = wb.sheet_by_name("children_cartoons") + expected_column_headers = [ + "children/cartoons/name", + "children/cartoons/why", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = cartoons_sheet.row_values(0) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + + characters_sheet = wb.sheet_by_name("children_cartoons_characters") + expected_column_headers = [ + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = characters_sheet.row_values(0) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close() @@ -1643,94 +1720,115 @@ def test_to_xls_export_respects_custom_field_delimiter(self): export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') + xls_file = NamedTemporaryFile(suffix=".xlsx") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check header columns - main_sheet = wb.get_sheet_by_name('childrens_survey') - expected_column_headers = ( - 'name', 'age', 'geo.geolocation', 'geo._geolocation_latitude', - 'geo._geolocation_longitude', 'geo._geolocation_altitude', - 'geo._geolocation_precision', 'tel.tel.office', - 'tel.tel.mobile', '_id', 'meta.instanceID', '_uuid', - '_submission_time', '_index', '_parent_index', - '_parent_table_name', '_tags', '_notes', '_version', - '_duration', '_submitted_by') - column_headers = tuple(main_sheet.values)[0] - self.assertEqual(sorted(column_headers), - sorted(expected_column_headers)) + main_sheet = wb.sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo.geolocation", + "geo._geolocation_latitude", + "geo._geolocation_longitude", + "geo._geolocation_altitude", + "geo._geolocation_precision", + "tel.tel.office", + "tel.tel.mobile", + "_id", + "meta.instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = main_sheet.row_values(0) + self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close() def test_get_valid_sheet_name_catches_duplicates(self): - work_sheets = {'childrens_survey': "Worksheet"} + work_sheets = {"childrens_survey": "Worksheet"} desired_sheet_name = "childrens_survey" expected_sheet_name = "childrens_survey1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheets) + desired_sheet_name, work_sheets + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_get_valid_sheet_name_catches_long_names(self): desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_lo" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, []) + desired_sheet_name, [] + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_get_valid_sheet_name_catches_long_duplicate_names(self): - work_sheet_titles = ['childrens_survey_with_a_very_lo'] + work_sheet_titles = ["childrens_survey_with_a_very_lo"] desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_l1" generated_sheet_name = ExportBuilder.get_valid_sheet_name( - desired_sheet_name, work_sheet_titles) + desired_sheet_name, work_sheet_titles + ) self.assertEqual(generated_sheet_name, expected_sheet_name) def test_to_xls_export_generates_valid_sheet_names(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xlsx'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') + xls_file = NamedTemporaryFile(suffix=".xlsx") filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets - expected_sheet_names = ['childrens_survey_with_a_very_lo', - 'childrens_survey_with_a_very_l1', - 'childrens_survey_with_a_very_l2', - 'childrens_survey_with_a_very_l3'] - self.assertEqual(wb.get_sheet_names(), expected_sheet_names) + expected_sheet_names = [ + "childrens_survey_with_a_very_lo", + "childrens_survey_with_a_very_l1", + "childrens_survey_with_a_very_l2", + "childrens_survey_with_a_very_l3", + ] + self.assertEqual(wb.sheet_names(), expected_sheet_names) xls_file.close() def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xlsx'), - default_name='childrens_survey_with_a_very_long_name') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), + default_name="childrens_survey_with_a_very_long_name", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix='.xlsx') + xls_file = NamedTemporaryFile(suffix=".xlsx") filename = xls_file.name export_builder.to_xls_export(filename, self.long_survey_data) xls_file.seek(0) wb = load_workbook(filename) # get the children's sheet - ws1 = wb['childrens_survey_with_a_very_l1'] + ws1 = wb["childrens_survey_with_a_very_l1"] # parent_table is in cell K2 - parent_table_name = ws1['K2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_lo' + parent_table_name = ws1["K2"].value + expected_parent_table_name = "childrens_survey_with_a_very_lo" self.assertEqual(parent_table_name, expected_parent_table_name) # get cartoons sheet - ws2 = wb['childrens_survey_with_a_very_l2'] - parent_table_name = ws2['G2'].value - expected_parent_table_name = 'childrens_survey_with_a_very_l1' + ws2 = wb["childrens_survey_with_a_very_l2"] + parent_table_name = ws2["G2"].value + expected_parent_table_name = "childrens_survey_with_a_very_l1" self.assertEqual(parent_table_name, expected_parent_table_name) xls_file.close() @@ -1747,15 +1845,12 @@ def test_type_conversion(self): "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", - "_geolocation": [ - "-1.2625482", - "36.7924794" - ], + "_geolocation": ["-1.2625482", "36.7924794"], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", - "really": "15:24:00.000+03" + "really": "15:24:00.000+03", } submission_2 = { @@ -1771,65 +1866,71 @@ def test_type_conversion(self): "amount": "", } - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xlsx'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xlsx"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} - data = dict_to_joined_export(submission_1, 1, indices, survey_name, - survey, submission_1) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['age'], int) - self.assertIsInstance(new_row['when'], datetime.date) - self.assertIsInstance(new_row['amount'], float) + data = dict_to_joined_export( + submission_1, 1, indices, survey_name, survey, submission_1 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["age"], int) + self.assertIsInstance(new_row["when"], datetime.date) + self.assertIsInstance(new_row["amount"], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} - data = dict_to_joined_export(submission_2, 1, indices, survey_name, - survey, submission_2) - new_row = export_builder.pre_process_row(data[survey_name], - export_builder.sections[0]) - self.assertIsInstance(new_row['amount'], basestring) - self.assertEqual(new_row['amount'], '') + data = dict_to_joined_export( + submission_2, 1, indices, survey_name, survey, submission_2 + ) + new_row = export_builder.pre_process_row( + data[survey_name], export_builder.sections[0] + ) + self.assertIsInstance(new_row["amount"], str) + self.assertEqual(new_row["amount"], "") def test_xls_convert_dates_before_1900(self): - survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xlsx'), - default_name='test_data_types') + survey = create_survey_from_xls( + viewer_fixture_path("test_data_types/test_data_types.xlsx"), + default_name="test_data_types", + ) export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ { - 'name': 'Abe', - 'when': '1899-07-03', + "name": "Abe", + "when": "1899-07-03", } ] # create export file - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, data) temp_xls_file.close() # this should error if there is a problem, not sure what to assert def test_convert_types(self): - val = '1' + val = "1" expected_val = 1 - converted_val = ExportBuilder.convert_type(val, 'int') + converted_val = ExportBuilder.convert_type(val, "int") self.assertIsInstance(converted_val, int) self.assertEqual(converted_val, expected_val) - val = '1.2' + val = "1.2" expected_val = 1.2 - converted_val = ExportBuilder.convert_type(val, 'decimal') + converted_val = ExportBuilder.convert_type(val, "decimal") self.assertIsInstance(converted_val, float) self.assertEqual(converted_val, expected_val) - val = '2012-06-23' + val = "2012-06-23" expected_val = datetime.date(2012, 6, 23) - converted_val = ExportBuilder.convert_type(val, 'date') + converted_val = ExportBuilder.convert_type(val, "date") self.assertIsInstance(converted_val, datetime.date) self.assertEqual(converted_val, expected_val) @@ -1838,7 +1939,7 @@ def test_to_sav_export(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -1855,47 +1956,47 @@ def test_to_sav_export(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + ) as reader: header = next(reader) rows = [r for r in reader] # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_to_sav_export_language(self): - survey = self._create_childrens_survey('childrens_survey_sw.xlsx') + survey = self._create_childrens_survey("childrens_survey_sw.xlsx") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -1912,91 +2013,94 @@ def test_to_sav_export_language(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): with SavReader( - os.path.join( - temp_dir, "{0}.sav".format(section)), - returnHeader=True) as reader: + os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + ) as reader: header = next(reader) rows = [r for r in reader] - if section != 'childrens_survey_sw': - section += '_sw' + if section != "childrens_survey_sw": + section += "_sw" # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) - if section == 'children_cartoons_charactors': - self.assertEqual(reader.valueLabels, { - 'good_or_evil': {'good': 'Good'} - }) + if section == "children_cartoons_charactors": + self.assertEqual( + reader.valueLabels, {"good_or_evil": {"good": "Good"}} + ) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_generate_field_title_truncated_titles(self): self._create_childrens_survey() - field_name = ExportBuilder.format_field_title("children/age", "/", - data_dictionary=self.dd, - remove_group_name=True) + field_name = ExportBuilder.format_field_title( + "children/age", "/", data_dictionary=self.dd, remove_group_name=True + ) expected_field_name = "age" self.assertEqual(field_name, expected_field_name) def test_generate_field_title_truncated_titles_select_multiple(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( - "children/fav_colors/red", "/", + "children/fav_colors/red", + "/", data_dictionary=self.dd, - remove_group_name=True + remove_group_name=True, ) expected_field_name = "fav_colors/red" self.assertEqual(field_name, expected_field_name) def test_xls_export_remove_group_name(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertTrue(data[u"fav_colors/red's"]) - self.assertTrue(data[u"fav_colors/blue's"]) - self.assertFalse(data[u"fav_colors/pink's"]) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) temp_xls_file.close() def test_zipped_csv_export_remove_group_name(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2005,100 +2109,109 @@ def test_zipped_csv_export_remove_group_name(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) def test_xls_export_with_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['name.first'], '3.1 Childs name') - self.assertEqual(labels['age'], '3.2 Child age') - self.assertEqual(labels['fav_colors/red\'s'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue\'s'], 'fav_colors/Blue') - self.assertEqual(labels['fav_colors/pink\'s'], 'fav_colors/Pink') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["name.first"], "3.1 Childs name") + self.assertEqual(labels["age"], "3.2 Child age") + self.assertEqual(labels["fav_colors/red's"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue's"], "fav_colors/Blue") + self.assertEqual(labels["fav_colors/pink's"], "fav_colors/Pink") data = dict([(r[0].value, r[2].value) for r in children_sheet.columns]) - self.assertEqual(data['name.first'], 'Mike') - self.assertEqual(data['age'], 5) - self.assertTrue(data['fav_colors/red\'s']) - self.assertTrue(data['fav_colors/blue\'s']) - self.assertFalse(data['fav_colors/pink\'s']) + self.assertEqual(data["name.first"], "Mike") + self.assertEqual(data["age"], 5) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) temp_xls_file.close() def test_xls_export_with_labels_only(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(data['3.1 Childs name'], 'Mike') - self.assertEqual(data['3.2 Child age'], 5) - self.assertTrue(data['fav_colors/Red']) - self.assertTrue(data['fav_colors/Blue']) - self.assertFalse(data['fav_colors/Pink']) + self.assertEqual(data["3.1 Childs name"], "Mike") + self.assertEqual(data["3.2 Child age"], 5) + self.assertTrue(data["fav_colors/Red"]) + self.assertTrue(data["fav_colors/Blue"]) + self.assertFalse(data["fav_colors/Pink"]) temp_xls_file.close() def test_zipped_csv_export_with_labels(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2107,50 +2220,67 @@ def test_zipped_csv_export_with_labels(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) - expected_headers = ['name.first', - 'age', - 'fav_colors', - 'fav_colors/red\'s', - 'fav_colors/blue\'s', - 'fav_colors/pink\'s', - 'ice_creams', - 'ice_creams/vanilla', - 'ice_creams/strawberry', - 'ice_creams/chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] - expected_labels = ['3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by'] + expected_headers = [ + "name.first", + "age", + "fav_colors", + "fav_colors/red's", + "fav_colors/blue's", + "fav_colors/pink's", + "ice_creams", + "ice_creams/vanilla", + "ice_creams/strawberry", + "ice_creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + expected_labels = [ + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) actual_labels = [h for h in rows[1]] self.assertEqual(sorted(actual_labels), sorted(expected_labels)) data = dict(zip(rows[0], rows[2])) - self.assertEqual(data['fav_colors/red\'s'], 'True') - self.assertEqual(data['fav_colors/blue\'s'], 'True') - self.assertEqual(data['fav_colors/pink\'s'], 'False') + self.assertEqual(data["fav_colors/red's"], "True") + self.assertEqual(data["fav_colors/blue's"], "True") + self.assertEqual(data["fav_colors/pink's"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2158,14 +2288,15 @@ def test_zipped_csv_export_with_labels_only(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_unicode.xlsx'), - default_name='childrens_survey_unicode') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_unicode.xlsx"), + default_name="childrens_survey_unicode", + ) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2174,36 +2305,42 @@ def test_zipped_csv_export_with_labels_only(self): zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists - self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "children.info.csv"))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv"), - encoding='utf-8') as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" + ) as csv_file: reader = csv.reader(csv_file) expected_headers = [ - '3.1 Childs name', - '3.2 Child age', - '3.3 Favorite Colors', - 'fav_colors/Red', - 'fav_colors/Blue', - 'fav_colors/Pink', - '3.3 Ice Creams', - 'ice_creams/Vanilla', - 'ice_creams/Strawberry', - 'ice_creams/Chocolate', '_id', - '_uuid', '_submission_time', '_index', - '_parent_table_name', '_parent_index', - '_tags', '_notes', '_version', - '_duration', '_submitted_by' + "3.1 Childs name", + "3.2 Child age", + "3.3 Favorite Colors", + "fav_colors/Red", + "fav_colors/Blue", + "fav_colors/Pink", + "3.3 Ice Creams", + "ice_creams/Vanilla", + "ice_creams/Strawberry", + "ice_creams/Chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", ] rows = [row for row in reader] actual_headers = [h for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) - self.assertEqual(data['fav_colors/Red'], 'True') - self.assertEqual(data['fav_colors/Blue'], 'True') - self.assertEqual(data['fav_colors/Pink'], 'False') + self.assertEqual(data["fav_colors/Red"], "True") + self.assertEqual(data["fav_colors/Blue"], "True") + self.assertEqual(data["fav_colors/Pink"], "False") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2214,7 +2351,7 @@ def test_to_sav_export_with_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) @@ -2231,32 +2368,41 @@ def test_to_sav_export_with_labels(self): outputs = [] for d in self.data: outputs.append( - dict_to_joined_export( - d, index, indices, survey_name, survey, d)) + dict_to_joined_export(d, index, indices, survey_name, survey, d) + ) index += 1 # check that each file exists self.assertTrue( - os.path.exists( - os.path.join(temp_dir, "{0}.sav".format(survey.name)))) + os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) + ) def _test_sav_file(section): sav_path = os.path.join(temp_dir, "{0}.sav".format(section)) - if section == 'children_survey': + if section == "children_survey": with SavHeaderReader(sav_path) as header: expected_labels = [ - '1. What is your name?', '2. How old are you?', - '4. Geo-location', '5.1 Office telephone', - '5.2 Mobile telephone', '_duration', '_id', - '_index', '_notes', '_parent_index', - '_parent_table_name', '_submission_time', - '_submitted_by', - '_tags', '_uuid', '_version', - 'geo/_geolocation_altitude', - 'geo/_geolocation_latitude', - 'geo/_geolocation_longitude', - 'geo/_geolocation_precision', - 'meta/instanceID' + "1. What is your name?", + "2. How old are you?", + "4. Geo-location", + "5.1 Office telephone", + "5.2 Mobile telephone", + "_duration", + "_id", + "_index", + "_notes", + "_parent_index", + "_parent_table_name", + "_submission_time", + "_submitted_by", + "_tags", + "_uuid", + "_version", + "geo/_geolocation_altitude", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_precision", + "meta/instanceID", ] labels = header.varLabels.values() self.assertEqual(sorted(expected_labels), sorted(labels)) @@ -2266,226 +2412,232 @@ def _test_sav_file(section): rows = [r for r in reader] # open comparison file - with SavReader(_logger_fixture_path( - 'spss', "{0}.sav".format(section)), - returnHeader=True) as fixture_reader: + with SavReader( + _logger_fixture_path("spss", "{0}.sav".format(section)), + returnHeader=True, + ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: - section_name = section['name'].replace('/', '_') + section_name = section["name"].replace("/", "_") _test_sav_file(section_name) def test_xls_export_with_english_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_en.xlsx'), - default_name='childrens_survey_en') - # no default_language is not set - self.assertEqual( - survey.to_json_dict().get('default_language'), 'default' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_en.xlsx"), + default_name="childrens_survey_en", ) + # no default_language is not set + self.assertEqual(survey.to_json_dict().get("default_language"), "default") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) childrens_survey_sheet = wb["childrens_survey_en"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. What is your name?') - self.assertEqual(labels['age'], '2. How old are you?') + labels = dict( + [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] + ) + self.assertEqual(labels["name"], "1. What is your name?") + self.assertEqual(labels["age"], "2. How old are you?") children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Red') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Blue') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["fav_colors/red"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Blue") temp_xls_file.close() def test_xls_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) childrens_survey_sheet = wb["childrens_survey_sw"] - labels = dict([(r[0].value, r[1].value) - for r in childrens_survey_sheet.columns]) - self.assertEqual(labels['name'], '1. Jina lako ni?') - self.assertEqual(labels['age'], '2. Umri wako ni?') + labels = dict( + [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] + ) + self.assertEqual(labels["name"], "1. Jina lako ni?") + self.assertEqual(labels["age"], "2. Umri wako ni?") children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) - for r in children_sheet.columns]) - self.assertEqual(labels['fav_colors/red'], 'fav_colors/Nyekundu') - self.assertEqual(labels['fav_colors/blue'], 'fav_colors/Bluu') + labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + self.assertEqual(labels["fav_colors/red"], "fav_colors/Nyekundu") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Bluu") temp_xls_file.close() def test_csv_export_with_swahili_labels(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') - # default_language is set to swahili - self.assertEqual( - survey.to_json_dict().get('default_language'), 'swahili' + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", ) + # default_language is set to swahili + self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") dd = DataDictionary() dd._survey = survey ordered_columns = OrderedDict() CSVDataFrameBuilder._build_ordered_columns(survey, ordered_columns) - ordered_columns['children/fav_colors/red'] = None - labels = get_labels_from_columns(ordered_columns, dd, '/') - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + ordered_columns["children/fav_colors/red"] = None + labels = get_labels_from_columns(ordered_columns, dd, "/") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) # use language provided in keyword argument - labels = get_labels_from_columns(ordered_columns, dd, '/', - language='english') - self.assertIn('1. What is your name?', labels) - self.assertIn('2. How old are you?', labels) - self.assertIn('fav_colors/Red', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="english") + self.assertIn("1. What is your name?", labels) + self.assertIn("2. How old are you?", labels) + self.assertIn("fav_colors/Red", labels) # use default language when language supplied does not exist - labels = get_labels_from_columns(ordered_columns, dd, '/', - language="Chinese") - self.assertIn('1. Jina lako ni?', labels) - self.assertIn('2. Umri wako ni?', labels) - self.assertIn('fav_colors/Nyekundu', labels) + labels = get_labels_from_columns(ordered_columns, dd, "/", language="Chinese") + self.assertIn("1. Jina lako ni?", labels) + self.assertIn("2. Umri wako ni?", labels) + self.assertIn("fav_colors/Nyekundu", labels) def test_select_multiples_choices(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_sw.xlsx'), - default_name='childrens_survey_sw') + survey = create_survey_from_xls( + _logger_fixture_path("childrens_survey_sw.xlsx"), + default_name="childrens_survey_sw", + ) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] self.assertNotEqual(child.children, []) choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'Nyekundu', - '_label_xpath': 'fav_colors/Nyekundu', - 'xpath': 'children/fav_colors/red', - 'title': 'children/fav_colors/red', - 'type': 'string', - 'label': 'fav_colors/Nyekundu' - }, { - '_label': 'Bluu', - '_label_xpath': 'fav_colors/Bluu', - 'xpath': 'children/fav_colors/blue', - 'title': 'children/fav_colors/blue', - 'type': 'string', 'label': 'fav_colors/Bluu' - }, { - '_label': 'Pink', - '_label_xpath': 'fav_colors/Pink', - 'xpath': 'children/fav_colors/pink', - 'title': 'children/fav_colors/pink', - 'type': 'string', 'label': 'fav_colors/Pink' - } + "_label": "Nyekundu", + "_label_xpath": "fav_colors/Nyekundu", + "xpath": "children/fav_colors/red", + "title": "children/fav_colors/red", + "type": "string", + "label": "fav_colors/Nyekundu", + }, + { + "_label": "Bluu", + "_label_xpath": "fav_colors/Bluu", + "xpath": "children/fav_colors/blue", + "title": "children/fav_colors/blue", + "type": "string", + "label": "fav_colors/Bluu", + }, + { + "_label": "Pink", + "_label_xpath": "fav_colors/Pink", + "xpath": "children/fav_colors/pink", + "title": "children/fav_colors/pink", + "type": "string", + "label": "fav_colors/Pink", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'children/fav_colors': [ - ('children/fav_colors/red', 'red', 'Nyekundu'), - ('children/fav_colors/blue', 'blue', 'Bluu'), - ('children/fav_colors/pink', 'pink', 'Pink') - ], 'children/ice.creams': [ - ('children/ice.creams/vanilla', 'vanilla', 'Vanilla'), - ('children/ice.creams/strawberry', 'strawberry', 'Strawberry'), - ('children/ice.creams/chocolate', 'chocolate', 'Chocolate'), - ] + "children/fav_colors": [ + ("children/fav_colors/red", "red", "Nyekundu"), + ("children/fav_colors/blue", "blue", "Bluu"), + ("children/fav_colors/pink", "pink", "Pink"), + ], + "children/ice.creams": [ + ("children/ice.creams/vanilla", "vanilla", "Vanilla"), + ("children/ice.creams/strawberry", "strawberry", "Strawberry"), + ("children/ice.creams/chocolate", "chocolate", "Chocolate"), + ], } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) def test_select_multiples_choices_with_choice_filter(self): - survey = create_survey_from_xls(_logger_fixture_path( - 'choice_filter.xlsx' - ), default_name='choice_filter') + survey = create_survey_from_xls( + _logger_fixture_path("choice_filter.xlsx"), default_name="choice_filter" + ) dd = DataDictionary() dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - child = [e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE][0] + child = [ + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE + ][0] choices = export_builder._get_select_mulitples_choices( - child, dd, ExportBuilder.GROUP_DELIMITER, - ExportBuilder.TRUNCATE_GROUP_TITLE + child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) expected_choices = [ { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Pierce', - '_label_xpath': 'county/Pierce', - 'label': 'county/Pierce', - 'title': 'county/pierce', - 'type': 'string', - 'xpath': 'county/pierce' + "_label": "Pierce", + "_label_xpath": "county/Pierce", + "label": "county/Pierce", + "title": "county/pierce", + "type": "string", + "xpath": "county/pierce", }, { - '_label': 'King', - '_label_xpath': 'county/King', - 'label': 'county/King', - 'title': 'county/king', - 'type': 'string', - 'xpath': 'county/king' + "_label": "King", + "_label_xpath": "county/King", + "label": "county/King", + "title": "county/king", + "type": "string", + "xpath": "county/king", }, { - '_label': 'Cameron', - '_label_xpath': 'county/Cameron', - 'label': 'county/Cameron', - 'title': 'county/cameron', - 'type': 'string', - 'xpath': 'county/cameron' - } + "_label": "Cameron", + "_label_xpath": "county/Cameron", + "label": "county/Cameron", + "title": "county/cameron", + "type": "string", + "xpath": "county/cameron", + }, ] self.assertEqual(choices, expected_choices) select_multiples = { - 'county': [ - ('county/king', 'king', 'King'), - ('county/pierce', 'pierce', 'Pierce'), - ('county/king', 'king', 'King'), - ('county/cameron', 'cameron', 'Cameron') + "county": [ + ("county/king", "king", "King"), + ("county/pierce", "pierce", "Pierce"), + ("county/king", "king", "King"), + ("county/cameron", "cameron", "Cameron"), ] } - self.assertEqual(CSVDataFrameBuilder._collect_select_multiples(dd), - select_multiples) + self.assertEqual( + CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples + ) def test_string_to_date_with_xls_validation(self): # test "2016-11-02" @@ -2505,36 +2657,36 @@ def _create_osm_survey(self): """ # publish form - osm_fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'apps', 'api', - 'tests', 'fixtures', 'osm') - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + osm_fixtures_dir = os.path.join( + settings.PROJECT_ROOT, "apps", "api", "tests", "fixtures", "osm" + ) + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) # make submissions filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - paths = [os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") self._make_submission_w_attachment(submission_path, paths) survey = create_survey_from_xls( - xlsform_path, - default_name=xlsform_path.split('/')[-1].split('.')[0]) + xlsform_path, default_name=xlsform_path.split("/")[-1].split(".")[0] + ) return survey def test_zip_csv_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2551,9 +2703,9 @@ def test_zip_csv_export_has_submission_review_fields(self): self.assertIn(REVIEW_DATE, sorted(actual_headers)) self.assertIn(REVIEW_STATUS, sorted(actual_headers)) submission = rows[1] - self.assertEqual(submission[29], 'Rejected') - self.assertEqual(submission[30], 'Wrong Location') - self.assertEqual(submission[31], '2021-05-25T02:27:19') + self.assertEqual(submission[29], "Rejected") + self.assertEqual(submission[30], "Wrong Location") + self.assertEqual(submission[31], "2021-05-25T02:27:19") # check that red and blue are set to true shutil.rmtree(temp_dir) @@ -2561,13 +2713,13 @@ def test_xls_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in xls exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") export_builder.to_xls_export(temp_xls_file.name, self.osm_data) temp_xls_file.seek(0) wb = load_workbook(temp_xls_file.name) @@ -2579,21 +2731,21 @@ def test_xls_export_has_submission_review_fields(self): self.assertIn(REVIEW_COMMENT, sorted(xls_headers)) self.assertIn(REVIEW_DATE, sorted(xls_headers)) self.assertIn(REVIEW_STATUS, sorted(xls_headers)) - self.assertEqual(xls_data[29], 'Rejected') - self.assertEqual(xls_data[30], 'Wrong Location') - self.assertEqual(xls_data[31], '2021-05-25T02:27:19') + self.assertEqual(xls_data[29], "Rejected") + self.assertEqual(xls_data[30], "Wrong Location") + self.assertEqual(xls_data[31], "2021-05-25T02:27:19") def test_zipped_sav_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports """ - self._create_user_and_login('dave', '1234') + self._create_user_and_login("dave", "1234") survey = self._create_osm_survey() xform = self.xform export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2602,25 +2754,49 @@ def test_zipped_sav_has_submission_review_fields(self): zip_file.close() temp_zip_file.close() - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_review_comment', f'@{REVIEW_DATE}', '@_review_status', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_review_comment", + f"@{REVIEW_DATE}", + "@_review_status", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], b'Rejected') - self.assertEqual(rows[1][30], b'Wrong Location') - self.assertEqual(rows[1][31], b'2021-05-25T02:27:19') + self.assertEqual(rows[1][29], b"Rejected") + self.assertEqual(rows[1][30], b"Wrong Location") + self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") def test_zipped_csv_export_with_osm_data(self): """ @@ -2630,7 +2806,7 @@ def test_zipped_csv_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2644,24 +2820,44 @@ def test_zipped_csv_export_with_osm_data(self): rows = [row for row in reader] expected_column_headers = [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta/instanceID', '_id', '_uuid', - '_submission_time', '_index', '_parent_table_name', - '_parent_index', '_tags', '_notes', '_version', '_duration', - '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', - 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', - 'osm_road:way:id', 'osm_building:building', - 'osm_building:building:levels', 'osm_building:ctr:lat', - 'osm_building:ctr:lon', 'osm_building:name', - 'osm_building:way:id'] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta/instanceID", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_table_name", + "_parent_index", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + "osm_road:ctr:lat", + "osm_road:ctr:lon", + "osm_road:highway", + "osm_road:lanes", + "osm_road:name", + "osm_road:way:id", + "osm_building:building", + "osm_building:building:levels", + "osm_building:ctr:lat", + "osm_building:ctr:lon", + "osm_building:name", + "osm_building:way:id", + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], '1424308569120.jpg') - self.assertEqual(rows[1][1], 'OSMWay234134797.osm') - self.assertEqual(rows[1][2], '23.708174238006087') - self.assertEqual(rows[1][4], 'tertiary') - self.assertEqual(rows[1][6], 'Patuatuli Road') - self.assertEqual(rows[1][13], 'kol') + self.assertEqual(rows[1][0], "1424308569120.jpg") + self.assertEqual(rows[1][1], "OSMWay234134797.osm") + self.assertEqual(rows[1][2], "23.708174238006087") + self.assertEqual(rows[1][4], "tertiary") + self.assertEqual(rows[1][6], "Patuatuli Road") + self.assertEqual(rows[1][13], "kol") def test_zipped_sav_export_with_osm_data(self): """ @@ -2669,25 +2865,27 @@ def test_zipped_sav_export_with_osm_data(self): """ survey = self._create_osm_survey() xform = self.xform - osm_data = [{ - 'photo': '1424308569120.jpg', - 'osm_road': 'OSMWay234134797.osm', - 'osm_building': 'OSMWay34298972.osm', - 'fav_color': 'red', - 'osm_road_ctr_lat': '23.708174238006087', - 'osm_road_ctr_lon': '90.40946505581161', - 'osm_road_highway': 'tertiary', - 'osm_road_lanes': '2', - 'osm_road_name': 'Patuatuli Road', - 'osm_building_building': 'yes', - 'osm_building_building_levels': '4', - 'osm_building_ctr_lat': '23.707316084046038', - 'osm_building_ctr_lon': '90.40849938337506', - 'osm_building_name': 'kol' - }] + osm_data = [ + { + "photo": "1424308569120.jpg", + "osm_road": "OSMWay234134797.osm", + "osm_building": "OSMWay34298972.osm", + "fav_color": "red", + "osm_road_ctr_lat": "23.708174238006087", + "osm_road_ctr_lon": "90.40946505581161", + "osm_road_highway": "tertiary", + "osm_road_lanes": "2", + "osm_road_name": "Patuatuli Road", + "osm_building_building": "yes", + "osm_building_building_levels": "4", + "osm_building_ctr_lat": "23.707316084046038", + "osm_building_ctr_lon": "90.40849938337506", + "osm_building_name": "kol", + } + ] export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix='.zip') + temp_zip_file = NamedTemporaryFile(suffix=".zip") export_builder.to_zipped_sav(temp_zip_file.name, osm_data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() @@ -2696,27 +2894,49 @@ def test_zipped_sav_export_with_osm_data(self): zip_file.close() temp_zip_file.close() - with SavReader(os.path.join(temp_dir, "osm.sav"), - returnHeader=True) as reader: + with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = [r for r in reader] - expected_column_headers = [x.encode('utf-8') for x in [ - 'photo', 'osm_road', 'osm_building', 'fav_color', - 'form_completed', 'meta.instanceID', '@_id', '@_uuid', - '@_submission_time', '@_index', '@_parent_table_name', - '@_parent_index', '@_tags', '@_notes', '@_version', - '@_duration', '@_submitted_by', 'osm_road_ctr_lat', - 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', - 'osm_road_name', 'osm_road_way_id', 'osm_building_building', - 'osm_building_building_levels', 'osm_building_ctr_lat', - 'osm_building_ctr_lon', 'osm_building_name', - 'osm_building_way_id']] + expected_column_headers = [ + x.encode("utf-8") + for x in [ + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", + ] + ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], b'1424308569120.jpg') - self.assertEqual(rows[1][1], b'OSMWay234134797.osm') - self.assertEqual(rows[1][2], b'23.708174238006087') - self.assertEqual(rows[1][4], b'tertiary') - self.assertEqual(rows[1][6], b'Patuatuli Road') - self.assertEqual(rows[1][13], b'kol') + self.assertEqual(rows[1][0], b"1424308569120.jpg") + self.assertEqual(rows[1][1], b"OSMWay234134797.osm") + self.assertEqual(rows[1][2], b"23.708174238006087") + self.assertEqual(rows[1][4], b"tertiary") + self.assertEqual(rows[1][6], b"Patuatuli Road") + self.assertEqual(rows[1][13], b"kol") def test_show_choice_labels(self): """ @@ -2734,24 +2954,19 @@ def test_show_choice_labels(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mango']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango"]] self.assertEqual(expected_result, result) @@ -2772,25 +2987,20 @@ def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True - export_builder.language = 'French' + export_builder.language = "French" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], ['Maria', 25, 'Mangue']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue"]] self.assertEqual(expected_result, result) @@ -2811,26 +3021,20 @@ def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) @@ -2852,26 +3056,20 @@ def test_show_choice_labels_select_multiple_1(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mango Orange']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mango Orange"]] self.assertEqual(expected_result, result) @@ -2893,29 +3091,24 @@ def test_show_choice_labels_select_multiple_2(self): | | fruits | 2 | Orange | | | fruits | 3 | Apple | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 2' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 2"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mango', 'fruit/Orange', - 'fruit/Apple'], - ['Maria', 25, 'Mango Orange', 'Mango', 'Orange', None]] + ["name", "age", "fruit", "fruit/Mango", "fruit/Orange", "fruit/Apple"], + ["Maria", 25, "Mango Orange", "Mango", "Orange", None], + ] self.maxDiff = None self.assertEqual(expected_result, result) @@ -2938,27 +3131,21 @@ def test_show_choice_labels_select_multiple_language(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:3]] - for row in children_sheet.rows] + result = [[col.value for col in row[:3]] for row in children_sheet.rows] temp_xls_file.close() - expected_result = [['name', 'age', 'fruit'], - ['Maria', 25, 'Mangue Pomme']] + expected_result = [["name", "age", "fruit"], ["Maria", 25, "Mangue Pomme"]] self.assertEqual(expected_result, result) @@ -2980,29 +3167,24 @@ def test_show_choice_labels_select_multiple_language_1(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', True, False, True]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", True, False, True], + ] self.assertEqual(expected_result, result) @@ -3025,30 +3207,25 @@ def test_show_choice_labels_select_multiple_language_2(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.BINARY_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 1, 0, 1]] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", 1, 0, 1], + ] self.assertEqual(expected_result, result) @@ -3071,30 +3248,25 @@ def test_show_choice_labels_select_multiple_language_3(self): | | fruits | 2 | Orange | Orange | | | fruits | 3 | Apple | Pomme | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = True export_builder.VALUE_SELECT_MULTIPLES = True - export_builder.language = 'Fr' + export_builder.language = "Fr" export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') - data = [{ - 'name': 'Maria', - 'age': 25, - 'fruit': '1 3' - }] # yapf: disable + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") + data = [{"name": "Maria", "age": 25, "fruit": "1 3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:6]] - for row in children_sheet.rows] + result = [[col.value for col in row[:6]] for row in children_sheet.rows] temp_xls_file.close() expected_result = [ - ['name', 'age', 'fruit', 'fruit/Mangue', 'fruit/Orange', - 'fruit/Pomme'], - ['Maria', 25, 'Mangue Pomme', 'Mangue', None, 'Pomme']] + ["name", "age", "fruit", "fruit/Mangue", "fruit/Orange", "fruit/Pomme"], + ["Maria", 25, "Mangue Pomme", "Mangue", None, "Pomme"], + ] self.assertEqual(expected_result, result) @@ -3127,27 +3299,24 @@ def test_mulsel_export_with_label_choices(self): | | primary | 9 | Other bank | 9 | """ - survey = self.md_to_pyxform_survey(md_xform, {'name': 'data'}) + survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) export_builder = ExportBuilder() export_builder.SHOW_CHOICE_LABELS = True export_builder.SPLIT_SELECT_MULTIPLES = False export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - data = [{ - 'banks_deal': '1 2 3 4', - 'primary_bank': '3' - }] # yapf: disable + data = [{"banks_deal": "1 2 3 4", "primary_bank": "3"}] # yapf: disable export_builder.to_xls_export(temp_xls_file, data) temp_xls_file.seek(0) children_sheet = load_workbook(temp_xls_file)["data"] self.assertTrue(children_sheet) - result = [[col.value for col in row[:2]] - for row in children_sheet.rows] + result = [[col.value for col in row[:2]] for row in children_sheet.rows] expected_result = [ - [u'banks_deal', u'primary_bank'], - [u'KCB Equity Co-operative CBA', u'Co-operative']] + ["banks_deal", "primary_bank"], + ["KCB Equity Co-operative CBA", "Co-operative"], + ] temp_xls_file.close() self.assertEqual(result, expected_result) diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index a411f4ae11..6aacfec896 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -5,7 +5,6 @@ from builtins import str as text from collections import OrderedDict -from past.builtins import basestring from django.db.utils import DataError from django.http import Http404 @@ -13,41 +12,45 @@ from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.xform import XForm -from onadata.libs.data.query import \ - get_form_submissions_aggregated_by_select_one +from onadata.libs.data.query import get_form_submissions_aggregated_by_select_one from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.data.query import get_form_submissions_grouped_by_select_one from onadata.libs.utils import common_tags # list of fields we can chart CHART_FIELDS = [ - 'select one', 'integer', 'decimal', 'date', 'datetime', 'start', 'end', - 'today' + "select one", + "integer", + "decimal", + "date", + "datetime", + "start", + "end", + "today", ] # numeric, categorized DATA_TYPE_MAP = { - 'integer': 'numeric', - 'decimal': 'numeric', - 'datetime': 'time_based', - 'date': 'time_based', - 'start': 'time_based', - 'end': 'time_based', - 'today': 'time_based', - 'calculate': 'numeric', + "integer": "numeric", + "decimal": "numeric", + "datetime": "time_based", + "date": "time_based", + "start": "time_based", + "end": "time_based", + "today": "time_based", + "calculate": "numeric", } FIELD_DATA_MAP = { - common_tags.SUBMISSION_TIME: - ('Submission Time', '_submission_time', 'datetime'), - common_tags.SUBMITTED_BY: ('Submission By', '_submitted_by', 'text'), - common_tags.DURATION: ('Duration', '_duration', 'integer') + common_tags.SUBMISSION_TIME: ("Submission Time", "_submission_time", "datetime"), + common_tags.SUBMITTED_BY: ("Submission By", "_submitted_by", "text"), + common_tags.DURATION: ("Duration", "_duration", "integer"), } CHARTS_PER_PAGE = 20 POSTGRES_ALIAS_LENGTH = 63 -timezone_re = re.compile(r'(.+)\+(\d+)') +timezone_re = re.compile(r"(.+)\+(\d+)") def utc_time_string_for_javascript(date_string): @@ -62,13 +65,13 @@ def utc_time_string_for_javascript(date_string): match = timezone_re.match(date_string) if not match: raise ValueError( - "{} fos not match the format 2014-01-16T12:07:23.322+03".format( - date_string)) + "{} fos not match the format 2014-01-16T12:07:23.322+03".format(date_string) + ) date_time = match.groups()[0] tz = match.groups()[1] if len(tz) == 2: - tz += '00' + tz += "00" elif len(tz) != 4: raise ValueError("len of {} must either be 2 or 4") @@ -77,8 +80,8 @@ def utc_time_string_for_javascript(date_string): def find_choice_label(choices, string): for choice in choices: - if choice['name'] == string: - return choice['label'] + if choice["name"] == string: + return choice["label"] def get_field_choices(field, xform): @@ -88,13 +91,13 @@ def get_field_choices(field, xform): :param xform: :return: Form field choices """ - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") - if isinstance(field, basestring): + if isinstance(field, str): choices = choices.get(field) - elif 'name' in field and field.name in choices: + elif "name" in field and field.name in choices: choices = choices.get(field.name) - elif 'itemset' in field: + elif "itemset" in field: choices = choices.get(field.itemset) return choices @@ -120,9 +123,7 @@ def get_choice_label(choices, string): labels.append(label) else: # Try to get labels by splitting the string - labels = [ - find_choice_label(choices, name) for name in string.split(" ") - ] + labels = [find_choice_label(choices, name) for name in string.split(" ")] # If any split string does not have a label it is not a multiselect # but a missing label, use string @@ -141,34 +142,29 @@ def _flatten_multiple_dict_into_one(field_name, group_by_name, data): # truncate field name to 63 characters to fix #354 truncated_field_name = field_name[0:POSTGRES_ALIAS_LENGTH] truncated_group_by_name = group_by_name[0:POSTGRES_ALIAS_LENGTH] - final = [{ - truncated_field_name: b, - 'items': [] - } for b in list({a.get(truncated_field_name) - for a in data})] + final = [ + {truncated_field_name: b, "items": []} + for b in list({a.get(truncated_field_name) for a in data}) + ] for a in data: for b in final: if a.get(truncated_field_name) == b.get(truncated_field_name): - b['items'].append({ - truncated_group_by_name: - a.get(truncated_group_by_name), - 'count': - a.get('count') - }) + b["items"].append( + { + truncated_group_by_name: a.get(truncated_group_by_name), + "count": a.get("count"), + } + ) return final -def _use_labels_from_field_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_field_name(field_name, field, data_type, data, choices=None): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized' and field_name != common_tags.SUBMITTED_BY: + if data_type == "categorized" and field_name != common_tags.SUBMITTED_BY: if data: if field.children: choices = field.children @@ -176,60 +172,54 @@ def _use_labels_from_field_name(field_name, for item in data: if truncated_name in item: item[truncated_name] = get_choice_label( - choices, item[truncated_name]) + choices, item[truncated_name] + ) for item in data: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def _use_labels_from_group_by_name(field_name, - field, - data_type, - data, - choices=None): +def _use_labels_from_group_by_name(field_name, field, data_type, data, choices=None): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] - if data_type == 'categorized': + if data_type == "categorized": if data: if field.children: choices = field.children for item in data: - if 'items' in item: - for i in item.get('items'): - i[truncated_name] = get_choice_label(choices, - i[truncated_name]) + if "items" in item: + for i in item.get("items"): + i[truncated_name] = get_choice_label(choices, i[truncated_name]) else: - item[truncated_name] = \ - get_choice_label(choices, item[truncated_name]) + item[truncated_name] = get_choice_label( + choices, item[truncated_name] + ) for item in data: - if 'items' in item: - for i in item.get('items'): + if "items" in item: + for i in item.get("items"): if field_name != truncated_name: i[field_name] = i[truncated_name] - del (i[truncated_name]) + del i[truncated_name] else: if field_name != truncated_name: item[field_name] = item[truncated_name] - del (item[truncated_name]) + del item[truncated_name] return data -def build_chart_data_for_field(xform, - field, - language_index=0, - choices=None, - group_by=None, - data_view=None): +def build_chart_data_for_field( + xform, field, language_index=0, choices=None, group_by=None, data_view=None +): # check if its the special _submission_time META - if isinstance(field, basestring): + if isinstance(field, str): field_label, field_xpath, field_type = FIELD_DATA_MAP.get(field) else: # TODO: merge choices with results and set 0's on any missing fields, @@ -239,96 +229,106 @@ def build_chart_data_for_field(xform, field_xpath = field.get_abbreviated_xpath() field_type = field.type - data_type = DATA_TYPE_MAP.get(field_type, 'categorized') - field_name = field.name if not isinstance(field, basestring) else field + data_type = DATA_TYPE_MAP.get(field_type, "categorized") + field_name = field.name if not isinstance(field, str) else field if group_by and isinstance(group_by, list): group_by_name = [ - g.get_abbreviated_xpath() if not isinstance(g, basestring) else g - for g in group_by + g.get_abbreviated_xpath() if not isinstance(g, str) else g for g in group_by ] result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) elif group_by: - group_by_name = group_by.get_abbreviated_xpath() \ - if not isinstance(group_by, basestring) else group_by - - if (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - isinstance(group_by, six.string_types): + group_by_name = ( + group_by.get_abbreviated_xpath() + if not isinstance(group_by, str) + else group_by + ) + + if ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and isinstance(group_by, six.string_types): result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - elif field_type in common_tags.NUMERIC_LIST and \ - isinstance(group_by, six.string_types): + xform, field_xpath, group_by_name, field_name, data_view + ) + elif field_type in common_tags.NUMERIC_LIST and isinstance( + group_by, six.string_types + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) - elif (field_type == common_tags.SELECT_ONE or - field_name == common_tags.SUBMITTED_BY) and \ - group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, field_name, group_by_name, data_view + ) + elif ( + field_type == common_tags.SELECT_ONE + or field_name == common_tags.SUBMITTED_BY + ) and group_by.type == common_tags.SELECT_ONE: result = get_form_submissions_grouped_by_select_one( - xform, field_xpath, group_by_name, field_name, data_view) - - result = _flatten_multiple_dict_into_one(field_name, group_by_name, - result) - elif field_type in common_tags.NUMERIC_LIST \ - and group_by.type == common_tags.SELECT_ONE: + xform, field_xpath, group_by_name, field_name, data_view + ) + + result = _flatten_multiple_dict_into_one(field_name, group_by_name, result) + elif ( + field_type in common_tags.NUMERIC_LIST + and group_by.type == common_tags.SELECT_ONE + ): result = get_form_submissions_aggregated_by_select_one( - xform, field_xpath, field_name, group_by_name, data_view) + xform, field_xpath, field_name, group_by_name, data_view + ) else: - raise ParseError('Cannot group by %s' % group_by_name) + raise ParseError("Cannot group by %s" % group_by_name) else: - result = get_form_submissions_grouped_by_field(xform, field_xpath, - field_name, data_view) + result = get_form_submissions_grouped_by_field( + xform, field_xpath, field_name, data_view + ) result = _use_labels_from_field_name( - field_name, field, data_type, result, choices=choices) + field_name, field, data_type, result, choices=choices + ) - if group_by and not isinstance(group_by, six.string_types + (list, )): - group_by_data_type = DATA_TYPE_MAP.get(group_by.type, 'categorized') + if group_by and not isinstance(group_by, six.string_types + (list,)): + group_by_data_type = DATA_TYPE_MAP.get(group_by.type, "categorized") grp_choices = get_field_choices(group_by, xform) result = _use_labels_from_group_by_name( - group_by_name, - group_by, - group_by_data_type, - result, - choices=grp_choices) + group_by_name, group_by, group_by_data_type, result, choices=grp_choices + ) elif group_by and isinstance(group_by, list): for g in group_by: if isinstance(g, six.string_types): continue - group_by_data_type = DATA_TYPE_MAP.get(g.type, 'categorized') + group_by_data_type = DATA_TYPE_MAP.get(g.type, "categorized") grp_choices = get_field_choices(g, xform) result = _use_labels_from_group_by_name( g.get_abbreviated_xpath(), g, group_by_data_type, result, - choices=grp_choices) + choices=grp_choices, + ) if not group_by: - result = sorted(result, key=lambda d: d['count']) + result = sorted(result, key=lambda d: d["count"]) # for date fields, strip out None values - if data_type == 'time_based': + if data_type == "time_based": result = [r for r in result if r.get(field_name) is not None] # for each check if it matches the timezone regexp and convert for js for r in result: if timezone_re.match(r[field_name]): try: - r[field_name] = utc_time_string_for_javascript( - r[field_name]) + r[field_name] = utc_time_string_for_javascript(r[field_name]) except ValueError: pass return { - 'data': result, - 'data_type': data_type, - 'field_label': field_label, - 'field_xpath': field_xpath, - 'field_name': field_name, - 'field_type': field_type, - 'grouped_by': group_by_name if group_by else None + "data": result, + "data_type": data_type, + "field_label": field_label, + "field_xpath": field_xpath, + "field_name": field_name, + "field_type": field_type, + "grouped_by": group_by_name if group_by else None, } @@ -355,8 +355,7 @@ def build_chart_data(xform, language_index=0, page=0): fields = fields[start:end] return [ - build_chart_data_for_field(xform, field, language_index) - for field in fields + build_chart_data_for_field(xform, field, language_index) for field in fields ] @@ -379,17 +378,15 @@ def build_chart_data_from_widget(widget, language_index=0): fields = [e for e in xform.survey_elements if e.name == field_name] if len(fields) == 0: - raise ParseError("Field %s does not not exist on the form" % - field_name) + raise ParseError("Field %s does not not exist on the form" % field_name) field = fields[0] - choices = xform.survey.get('choices') + choices = xform.survey.get("choices") if choices: choices = choices.get(field_name) try: - data = build_chart_data_for_field( - xform, field, language_index, choices=choices) + data = build_chart_data_for_field(xform, field, language_index, choices=choices) except DataError as e: raise ParseError(text(e)) @@ -408,8 +405,7 @@ def _get_field_from_field_fn(field_str, xform, field_fn): # use specified field to get summary fields = [e for e in xform.survey_elements if field_fn(e) == field_str] if len(fields) == 0: - raise Http404("Field %s does not not exist on the form" % - field_str) + raise Http404("Field %s does not not exist on the form" % field_str) field = fields[0] return field @@ -420,7 +416,8 @@ def get_field_from_field_name(field_name, xform): def get_field_from_field_xpath(field_xpath, xform): return _get_field_from_field_fn( - field_xpath, xform, lambda x: x.get_abbreviated_xpath()) + field_xpath, xform, lambda x: x.get_abbreviated_xpath() + ) def get_field_label(field, language_index=0): @@ -435,12 +432,9 @@ def get_field_label(field, language_index=0): return field_label -def get_chart_data_for_field(field_name, - xform, - accepted_format, - group_by, - field_xpath=None, - data_view=None): +def get_chart_data_for_field( + field_name, xform, accepted_format, group_by, field_xpath=None, data_view=None +): """ Get chart data for a given xlsform field. """ @@ -450,10 +444,9 @@ def get_chart_data_for_field(field_name, field = get_field_from_field_name(field_name, xform) if group_by: - if len(group_by.split(',')) > 1: + if len(group_by.split(",")) > 1: group_by = [ - get_field_from_field_xpath(g, xform) - for g in group_by.split(',') + get_field_from_field_xpath(g, xform) for g in group_by.split(",") ] else: group_by = get_field_from_field_xpath(group_by, xform) @@ -465,21 +458,18 @@ def get_chart_data_for_field(field_name, try: data = build_chart_data_for_field( - xform, - field, - choices=choices, - group_by=group_by, - data_view=data_view) + xform, field, choices=choices, group_by=group_by, data_view=data_view + ) except DataError as e: raise ParseError(text(e)) else: - if accepted_format == 'json' or not accepted_format: + if accepted_format == "json" or not accepted_format: xform = xform.pk - elif accepted_format == 'html' and 'data' in data: - for item in data['data']: + elif accepted_format == "html" and "data" in data: + for item in data["data"]: if isinstance(item[field_name], list): - item[field_name] = ', '.join(item[field_name]) + item[field_name] = ", ".join(item[field_name]) - data.update({'xform': xform}) + data.update({"xform": xform}) return data diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index 4abfece1e0..a6e7a1fd79 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -11,7 +11,6 @@ import traceback import uuid from io import BytesIO -from past.builtins import basestring from django.conf import settings from django.core.mail import mail_admins @@ -21,7 +20,7 @@ import six from raven.contrib.django.raven_compat.models import client -TRUE_VALUES = ['TRUE', 'T', '1', 1] +TRUE_VALUES = ["TRUE", "T", "1", 1] def str_to_bool(str_var): @@ -41,17 +40,16 @@ def get_boolean_value(str_var, default=None): """ Converts a string into boolean """ - if isinstance(str_var, basestring) and \ - str_var.lower() in ['true', 'false']: + if isinstance(str_var, str) and str_var.lower() in ["true", "false"]: return str_to_bool(str_var) return str_var if default else False def get_uuid(hex_only: bool = True): - ''' + """ Return UUID4 hex value - ''' + """ return uuid.uuid4().hex if hex_only else str(uuid.uuid4()) @@ -65,18 +63,19 @@ def report_exception(subject, info, exc_info=None): if exc_info: cls, err = exc_info[:2] - message = _(u"Exception in request:" - u" %(class)s: %(error)s")\ - % {'class': cls.__name__, 'error': err} - message += u"".join(traceback.format_exception(*exc_info)) + message = _("Exception in request:" " %(class)s: %(error)s") % { + "class": cls.__name__, + "error": err, + } + message += "".join(traceback.format_exception(*exc_info)) # send to sentry try: client.captureException(exc_info) except Exception: # pylint: disable=broad-except - logging.exception(_(u'Sending to Sentry failed.')) + logging.exception(_("Sending to Sentry failed.")) else: - message = u"%s" % info + message = "%s" % info if settings.DEBUG or settings.TESTING_MODE: sys.stdout.write("Subject: %s\n" % subject) @@ -89,12 +88,12 @@ def filename_from_disposition(content_disposition): """ Gets a filename from the given content disposition header. """ - filename_pos = content_disposition.index('filename=') + filename_pos = content_disposition.index("filename=") if filename_pos == -1: raise Exception('"filename=" not found in content disposition file') - return content_disposition[filename_pos + len('filename='):] + return content_disposition[filename_pos + len("filename=") :] def get_response_content(response, decode=True): @@ -106,7 +105,7 @@ def get_response_content(response, decode=True): :param response: The response to extract content from. :param decode: If true decode as utf-8, default True. """ - contents = '' + contents = "" if response.streaming: actual_content = BytesIO() for content in response.streaming_content: @@ -117,7 +116,7 @@ def get_response_content(response, decode=True): contents = response.content if decode: - return contents.decode('utf-8') + return contents.decode("utf-8") else: return contents @@ -126,7 +125,7 @@ def json_stream(data, json_string): """ Generator function to stream JSON data """ - yield '[' + yield "[" try: data = data.__iter__() item = next(data) @@ -134,7 +133,7 @@ def json_stream(data, json_string): try: next_item = next(data) yield json_string(item) - yield ',' + yield "," item = next_item except StopIteration: yield json_string(item) @@ -142,7 +141,7 @@ def json_stream(data, json_string): except (AttributeError, StopIteration): pass finally: - yield ']' + yield "]" def retry(tries, delay=3, backoff=2): @@ -181,8 +180,9 @@ def function_retry(self, *args, **kwargs): else: return result # Last ditch effort run against master database - if len(getattr(settings, 'SLAVE_DATABASES', [])): + if len(getattr(settings, "SLAVE_DATABASES", [])): from multidb.pinning import use_master + with use_master: return func(self, *args, **kwargs) @@ -190,11 +190,12 @@ def function_retry(self, *args, **kwargs): return func(self, *args, **kwargs) return function_retry + return decorator_retry def merge_dicts(*dict_args): - """ Given any number of dicts, shallow copy and merge into a new dict, + """Given any number of dicts, shallow copy and merge into a new dict, precedence goes to key value pairs in latter dicts. """ result = {} @@ -206,8 +207,8 @@ def merge_dicts(*dict_args): def cmp_to_key(mycmp): - """ Convert a cmp= function into a key= function - """ + """Convert a cmp= function into a key= function""" + class K(object): def __init__(self, obj, *args): self.obj = obj @@ -229,4 +230,5 @@ def __ge__(self, other): def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 + return K diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index a2023d7597..2a8d99e24c 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -5,43 +5,64 @@ from django.conf import settings from django.db.models.query import QuerySet from django.utils.translation import ugettext as _ -from future.utils import iteritems -from past.builtins import basestring +from six import iteritems from pyxform.question import Question from pyxform.section import RepeatingSection, Section from onadata.apps.logger.models import OsmData from onadata.apps.logger.models.xform import XForm, question_types_to_exclude from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.apps.viewer.models.parsed_instance import (ParsedInstance, - query_data) +from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED, DELETEDAT, DURATION, - EDITED, GEOLOCATION, ID, MEDIA_ALL_RECEIVED, MEDIA_COUNT, NA_REP, - NOTES, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, SELECT_BIND_TYPE, REVIEW_DATE) -from onadata.libs.utils.export_builder import (get_choice_label, - get_value_or_attachment_uri, - track_task_progress) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + GEOLOCATION, + ID, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + NA_REP, + NOTES, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID_STRING, + REVIEW_STATUS, + REVIEW_COMMENT, + MULTIPLE_SELECT_TYPE, + SELECT_BIND_TYPE, + REVIEW_DATE, +) +from onadata.libs.utils.export_builder import ( + get_choice_label, + get_value_or_attachment_uri, + track_task_progress, +) from onadata.libs.utils.model_tools import get_columns_with_hxl # the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = u"select" -GEOPOINT_BIND_TYPE = u"geopoint" +MULTIPLE_SELECT_BIND_TYPE = "select" +GEOPOINT_BIND_TYPE = "geopoint" # column group delimiters -GROUP_DELIMITER_SLASH = '/' -GROUP_DELIMITER_DOT = '.' +GROUP_DELIMITER_SLASH = "/" +GROUP_DELIMITER_DOT = "." DEFAULT_GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] -DEFAULT_NA_REP = getattr(settings, 'NA_REP', NA_REP) +DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) # index tags -DEFAULT_OPEN_TAG = '[' -DEFAULT_CLOSE_TAG = ']' +DEFAULT_OPEN_TAG = "[" +DEFAULT_CLOSE_TAG = "]" DEFAULT_INDEX_TAGS = (DEFAULT_OPEN_TAG, DEFAULT_CLOSE_TAG) YES = 1 @@ -54,25 +75,23 @@ def remove_dups_from_list_maintain_order(lst): def get_prefix_from_xpath(xpath): xpath = str(xpath) - parts = xpath.rsplit('/', 1) + parts = xpath.rsplit("/", 1) if len(parts) == 1: return None elif len(parts) == 2: - return '%s/' % parts[0] + return "%s/" % parts[0] else: - raise ValueError( - '%s cannot be prefixed, it returns %s' % (xpath, str(parts))) + raise ValueError("%s cannot be prefixed, it returns %s" % (xpath, str(parts))) def get_labels_from_columns(columns, dd, group_delimiter, language=None): labels = [] for col in columns: elem = dd.get_survey_element(col) - label = dd.get_label(col, elem=elem, - language=language) if elem else col - if elem is not None and elem.type == '': + label = dd.get_label(col, elem=elem, language=language) if elem else col + if elem is not None and elem.type == "": label = group_delimiter.join([elem.parent.name, label]) - if label == '': + if label == "": label = elem.name labels.append(label) @@ -86,28 +105,35 @@ def get_column_names_only(columns, dd, group_delimiter): elem = dd.get_survey_element(col) if elem is None: new_col = col - elif elem.type != '': + elif elem.type != "": new_col = elem.name else: - new_col = DEFAULT_GROUP_DELIMITER.join([ - elem.parent.name, - elem.name - ]) + new_col = DEFAULT_GROUP_DELIMITER.join([elem.parent.name, elem.name]) new_columns.append(new_col) return new_columns -def write_to_csv(path, rows, columns, columns_with_hxl=None, - remove_group_name=False, dd=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, include_labels=False, - include_labels_only=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, language=None): - na_rep = getattr(settings, 'NA_REP', NA_REP) - encoding = 'utf-8-sig' if win_excel_utf8 else 'utf-8' - with open(path, 'wb') as csvfile: - writer = csv.writer(csvfile, encoding=encoding, lineterminator='\n') +def write_to_csv( + path, + rows, + columns, + columns_with_hxl=None, + remove_group_name=False, + dd=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + include_labels=False, + include_labels_only=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + language=None, +): + na_rep = getattr(settings, "NA_REP", NA_REP) + encoding = "utf-8-sig" if win_excel_utf8 else "utf-8" + with open(path, "wb") as csvfile: + writer = csv.writer(csvfile, encoding=encoding, lineterminator="\n") # Check if to truncate the group name prefix if not include_labels_only: @@ -126,12 +152,13 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, writer.writerow(new_cols) if include_labels or include_labels_only: - labels = get_labels_from_columns(columns, dd, group_delimiter, - language=language) + labels = get_labels_from_columns( + columns, dd, group_delimiter, language=language + ) writer.writerow(labels) if include_hxl and columns_with_hxl: - hxl_row = [columns_with_hxl.get(col, '') for col in columns] + hxl_row = [columns_with_hxl.get(col, "") for col in columns] hxl_row and writer.writerow(hxl_row) for i, row in enumerate(rows, start=1): @@ -142,29 +169,60 @@ def write_to_csv(path, rows, columns, columns_with_hxl=None, class AbstractDataFrameBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT, EDITED] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + EDITED, + ] # fields NOT within the form def that we want to include ADDITIONAL_COLUMNS = [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, VERSION, - DURATION, SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED] + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, + ] BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False """ Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML """ - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=True, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=True, include_reviews=False, - language=None): + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=True, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=True, + include_reviews=False, + language=None, + ): self.username = username self.id_string = id_string @@ -176,30 +234,39 @@ def __init__(self, username, id_string, filter_query=None, self.start = start self.end = end self.remove_group_name = remove_group_name - self.extra_columns = ( - self.ADDITIONAL_COLUMNS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.ADDITIONAL_COLUMNS + getattr( + settings, "EXTRA_COLUMNS", [] + ) if include_reviews: self.extra_columns = self.extra_columns + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] if xform: self.xform = xform else: - self.xform = XForm.objects.get(id_string=self.id_string, - user__username=self.username) + self.xform = XForm.objects.get( + id_string=self.id_string, user__username=self.username + ) self.include_labels = include_labels self.include_labels_only = include_labels_only self.include_images = include_images self.include_hxl = include_hxl self.win_excel_utf8 = win_excel_utf8 self.total_records = total_records - if index_tags != DEFAULT_INDEX_TAGS and \ - not isinstance(index_tags, (tuple, list)): - raise ValueError(_( - "Invalid option for repeat_index_tags: %s " - "expecting a tuple with opening and closing tags " - "e.g repeat_index_tags=('[', ']')" % index_tags)) + if index_tags != DEFAULT_INDEX_TAGS and not isinstance( + index_tags, (tuple, list) + ): + raise ValueError( + _( + "Invalid option for repeat_index_tags: %s " + "expecting a tuple with opening and closing tags " + "e.g repeat_index_tags=('[', ']')" % index_tags + ) + ) self.index_tags = index_tags self.show_choice_labels = show_choice_labels self.language = language @@ -208,46 +275,68 @@ def __init__(self, username, id_string, filter_query=None, def _setup(self): self.dd = self.xform - self.select_multiples = self._collect_select_multiples(self.dd, - self.language) + self.select_multiples = self._collect_select_multiples(self.dd, self.language) self.gps_fields = self._collect_gps_fields(self.dd) @classmethod def _fields_to_select(cls, dd): - return [c.get_abbreviated_xpath() - for c in dd.get_survey_elements() if isinstance(c, Question)] + return [ + c.get_abbreviated_xpath() + for c in dd.get_survey_elements() + if isinstance(c, Question) + ] @classmethod def _collect_select_multiples(cls, dd, language=None): select_multiples = [] select_multiple_elements = [ - e for e in dd.get_survey_elements_with_choices() - if e.bind.get('type') == SELECT_BIND_TYPE - and e.type == MULTIPLE_SELECT_TYPE + e + for e in dd.get_survey_elements_with_choices() + if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ] for e in select_multiple_elements: xpath = e.get_abbreviated_xpath() - choices = [(c.get_abbreviated_xpath(), c.name, - get_choice_label(c.label, dd, language)) - for c in e.children] + choices = [ + ( + c.get_abbreviated_xpath(), + c.name, + get_choice_label(c.label, dd, language), + ) + for c in e.children + ] is_choice_randomized = str_to_bool( - e.parameters and e.parameters.get('randomize')) - if ((not choices and e.choice_filter) or is_choice_randomized) \ - and e.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(e.itemset) - choices = [(u'/'.join([xpath, i.get('name')]), i.get('name'), - get_choice_label(i.get('label'), dd, language)) - for i in itemset] if itemset else choices + e.parameters and e.parameters.get("randomize") + ) + if ( + (not choices and e.choice_filter) or is_choice_randomized + ) and e.itemset: + itemset = dd.survey.to_json_dict()["choices"].get(e.itemset) + choices = ( + [ + ( + "/".join([xpath, i.get("name")]), + i.get("name"), + get_choice_label(i.get("label"), dd, language), + ) + for i in itemset + ] + if itemset + else choices + ) select_multiples.append((xpath, choices)) return dict(select_multiples) @classmethod - def _split_select_multiples(cls, record, select_multiples, - binary_select_multiples=False, - value_select_multiples=False, - show_choice_labels=False): - """ Prefix contains the xpath and slash if we are within a repeat so + def _split_select_multiples( + cls, + record, + select_multiples, + binary_select_multiples=False, + value_select_multiples=False, + show_choice_labels=False, + ): + """Prefix contains the xpath and slash if we are within a repeat so that we can figure out which select multiples belong to which repeat """ for key, choices in select_multiples.items(): @@ -257,31 +346,59 @@ def _split_select_multiples(cls, record, select_multiples, if key in record: # split selected choices by spaces and join by / to the # element's xpath - selections = ["%s/%s" % (key, r) - for r in record[key].split(" ")] + selections = ["%s/%s" % (key, r) for r in record[key].split(" ")] if value_select_multiples: - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - (label if show_choice_labels else - record[key].split()[selections.index(choice)]) - if choice in selections else None) - for choice, name, label in choices])) + record.update( + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + ( + label + if show_choice_labels + else record[key].split()[ + selections.index(choice) + ] + ) + if choice in selections + else None, + ) + for choice, name, label in choices + ] + ) + ) elif not binary_select_multiples: # add columns to record for every choice, with default # False and set to True for items in selections - record.update(dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - choice in selections) - for choice, name, label in choices])) + record.update( + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + choice in selections, + ) + for choice, name, label in choices + ] + ) + ) else: record.update( - dict([ - (choice.replace('/' + name, '/' + label) - if show_choice_labels else choice, - YES if choice in selections else NO) - for choice, name, label in choices])) + dict( + [ + ( + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice, + YES if choice in selections else NO, + ) + for choice, name, label in choices + ] + ) + ) # remove the column since we are adding separate columns # for each choice record.pop(key) @@ -292,40 +409,45 @@ def _split_select_multiples(cls, record, select_multiples, for list_item in record_item: if isinstance(list_item, dict): cls._split_select_multiples( - list_item, select_multiples, + list_item, + select_multiples, binary_select_multiples=binary_select_multiples, # noqa value_select_multiples=value_select_multiples, - show_choice_labels=show_choice_labels) + show_choice_labels=show_choice_labels, + ) return record @classmethod def _collect_gps_fields(cls, dd): - return [e.get_abbreviated_xpath() for e in dd.get_survey_elements() - if e.bind.get("type") == "geopoint"] + return [ + e.get_abbreviated_xpath() + for e in dd.get_survey_elements() + if e.bind.get("type") == "geopoint" + ] @classmethod def _tag_edit_string(cls, record): """ Turns a list of tags into a string representation. """ - if '_tags' in record: + if "_tags" in record: tags = [] - for tag in record['_tags']: - if ',' in tag and ' ' in tag: + for tag in record["_tags"]: + if "," in tag and " " in tag: tags.append('"%s"' % tag) else: tags.append(tag) - record.update({'_tags': u', '.join(sorted(tags))}) + record.update({"_tags": ", ".join(sorted(tags))}) @classmethod def _split_gps_fields(cls, record, gps_fields): updated_gps_fields = {} for (key, value) in iteritems(record): - if key in gps_fields and isinstance(value, basestring): + if key in gps_fields and isinstance(value, str): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key) gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) # hack, check if its a list and grab the object within that - parts = value.split(' ') + parts = value.split(" ") # TODO: check whether or not we can have a gps recording # from ODKCollect that has less than four components, # for now we are assuming that this is not the case. @@ -339,19 +461,24 @@ def _split_gps_fields(cls, record, gps_fields): cls._split_gps_fields(list_item, gps_fields) record.update(updated_gps_fields) - def _query_data(self, query='{}', start=0, - limit=ParsedInstance.DEFAULT_LIMIT, - fields='[]', count=False): + def _query_data( + self, + query="{}", + start=0, + limit=ParsedInstance.DEFAULT_LIMIT, + fields="[]", + count=False, + ): # query_data takes params as json strings # so we dumps the fields dictionary count_args = { - 'xform': self.xform, - 'query': query, - 'start': self.start, - 'end': self.end, - 'fields': '[]', - 'sort': '{}', - 'count': True + "xform": self.xform, + "query": query, + "start": self.start, + "end": self.end, + "fields": "[]", + "sort": "{}", + "count": True, } count_object = list(query_data(**count_args)) record_count = count_object[0]["count"] @@ -362,17 +489,17 @@ def _query_data(self, query='{}', start=0, return record_count else: query_args = { - 'xform': self.xform, - 'query': query, - 'fields': fields, - 'start': self.start, - 'end': self.end, + "xform": self.xform, + "query": query, + "fields": fields, + "start": self.start, + "end": self.end, # TODO: we might want to add this in for the user # to sepcify a sort order - 'sort': 'id', - 'start_index': start, - 'limit': limit, - 'count': False + "sort": "id", + "start_index": start, + "limit": limit, + "count": False, } cursor = query_data(**query_args) @@ -380,49 +507,88 @@ def _query_data(self, query='{}', start=0, class CSVDataFrameBuilder(AbstractDataFrameBuilder): - - def __init__(self, username, id_string, filter_query=None, - group_delimiter=DEFAULT_GROUP_DELIMITER, - split_select_multiples=True, binary_select_multiples=False, - start=None, end=None, remove_group_name=False, xform=None, - include_labels=False, include_labels_only=False, - include_images=False, include_hxl=False, - win_excel_utf8=False, total_records=None, - index_tags=DEFAULT_INDEX_TAGS, value_select_multiples=False, - show_choice_labels=False, include_reviews=False, - language=None): + def __init__( + self, + username, + id_string, + filter_query=None, + group_delimiter=DEFAULT_GROUP_DELIMITER, + split_select_multiples=True, + binary_select_multiples=False, + start=None, + end=None, + remove_group_name=False, + xform=None, + include_labels=False, + include_labels_only=False, + include_images=False, + include_hxl=False, + win_excel_utf8=False, + total_records=None, + index_tags=DEFAULT_INDEX_TAGS, + value_select_multiples=False, + show_choice_labels=False, + include_reviews=False, + language=None, + ): super(CSVDataFrameBuilder, self).__init__( - username, id_string, filter_query, group_delimiter, - split_select_multiples, binary_select_multiples, start, end, - remove_group_name, xform, include_labels, include_labels_only, - include_images, include_hxl, win_excel_utf8, total_records, - index_tags, value_select_multiples, - show_choice_labels, include_reviews, language) + username, + id_string, + filter_query, + group_delimiter, + split_select_multiples, + binary_select_multiples, + start, + end, + remove_group_name, + xform, + include_labels, + include_labels_only, + include_images, + include_hxl, + win_excel_utf8, + total_records, + index_tags, + value_select_multiples, + show_choice_labels, + include_reviews, + language, + ) self.ordered_columns = OrderedDict() - self.image_xpaths = [] if not self.include_images \ - else self.dd.get_media_survey_xpaths() + self.image_xpaths = ( + [] if not self.include_images else self.dd.get_media_survey_xpaths() + ) def _setup(self): super(CSVDataFrameBuilder, self)._setup() @classmethod - def _reindex(cls, key, value, ordered_columns, row, data_dictionary, - parent_prefix=None, - include_images=True, split_select_multiples=True, - index_tags=DEFAULT_INDEX_TAGS, show_choice_labels=False, - language=None): + def _reindex( + cls, + key, + value, + ordered_columns, + row, + data_dictionary, + parent_prefix=None, + include_images=True, + split_select_multiples=True, + index_tags=DEFAULT_INDEX_TAGS, + show_choice_labels=False, + language=None, + ): """ Flatten list columns by appending an index, otherwise return as is """ + def get_ordered_repeat_value(xpath, repeat_value): """ Return OrderedDict of repeats in the order in which they appear in the XForm. """ - children = data_dictionary.get_child_elements( - xpath, split_select_multiples) + children = data_dictionary.get_child_elements(xpath, split_select_multiples) item = OrderedDict() for elem in children: @@ -435,8 +601,11 @@ def get_ordered_repeat_value(xpath, repeat_value): d = {} # check for lists - if isinstance(value, list) and len(value) > 0 \ - and key not in [ATTACHMENTS, NOTES]: + if ( + isinstance(value, list) + and len(value) > 0 + and key not in [ATTACHMENTS, NOTES] + ): for index, item in enumerate(value): # start at 1 index += 1 @@ -452,69 +621,95 @@ def get_ordered_repeat_value(xpath, repeat_value): # "children/details/immunization/polio_1", # generate ["children", index, "immunization/polio_1"] if parent_prefix is not None: - _key = '/'.join( - parent_prefix + - key.split('/')[len(parent_prefix):]) - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=_key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(_key.split('/')):] + _key = "/".join( + parent_prefix + key.split("/")[len(parent_prefix) :] + ) + xpaths = [ + "{key}{open_tag}{index}{close_tag}".format( + key=_key, + open_tag=index_tags[0], + index=index, + close_tag=index_tags[1], + ) + ] + nested_key.split("/")[len(_key.split("/")) :] else: - xpaths = ['{key}{open_tag}{index}{close_tag}' - .format(key=key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1])] + \ - nested_key.split('/')[len(key.split('/')):] + xpaths = [ + "{key}{open_tag}{index}{close_tag}".format( + key=key, + open_tag=index_tags[0], + index=index, + close_tag=index_tags[1], + ) + ] + nested_key.split("/")[len(key.split("/")) :] # re-create xpath the split on / xpaths = "/".join(xpaths).split("/") new_prefix = xpaths[:-1] if isinstance(nested_val, list): # if nested_value is a list, rinse and repeat - d.update(cls._reindex( - nested_key, nested_val, - ordered_columns, row, data_dictionary, - new_prefix, - include_images=include_images, - split_select_multiples=split_select_multiples, - index_tags=index_tags, - show_choice_labels=show_choice_labels, - language=language)) + d.update( + cls._reindex( + nested_key, + nested_val, + ordered_columns, + row, + data_dictionary, + new_prefix, + include_images=include_images, + split_select_multiples=split_select_multiples, + index_tags=index_tags, + show_choice_labels=show_choice_labels, + language=language, + ) + ) else: # it can only be a scalar # collapse xpath - new_xpath = u"/".join(xpaths) + new_xpath = "/".join(xpaths) # check if this key exists in our ordered columns if key in list(ordered_columns): if new_xpath not in ordered_columns[key]: ordered_columns[key].append(new_xpath) d[new_xpath] = get_value_or_attachment_uri( - nested_key, nested_val, row, data_dictionary, + nested_key, + nested_val, + row, + data_dictionary, include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, + key, + value, + row, + data_dictionary, + include_images, show_choice_labels=show_choice_labels, - language=language) + language=language, + ) else: # anything that's not a list will be in the top level dict so its # safe to simply assign if key == NOTES: # Do not include notes - d[key] = u"" + d[key] = "" else: d[key] = get_value_or_attachment_uri( - key, value, row, data_dictionary, include_images, - show_choice_labels=show_choice_labels, language=language) + key, + value, + row, + data_dictionary, + include_images, + show_choice_labels=show_choice_labels, + language=language, + ) return d @classmethod - def _build_ordered_columns(cls, survey_element, ordered_columns, - is_repeating_section=False): + def _build_ordered_columns( + cls, survey_element, ordered_columns, is_repeating_section=False + ): """ Build a flat ordered dict of column groups @@ -528,11 +723,12 @@ def _build_ordered_columns(cls, survey_element, ordered_columns, if isinstance(child, RepeatingSection): ordered_columns[child.get_abbreviated_xpath()] = [] child_is_repeating = True - cls._build_ordered_columns(child, ordered_columns, - child_is_repeating) - elif isinstance(child, Question) and not \ - question_types_to_exclude(child.type) and not\ - is_repeating_section: # if is_repeating_section, + cls._build_ordered_columns(child, ordered_columns, child_is_repeating) + elif ( + isinstance(child, Question) + and not question_types_to_exclude(child.type) + and not is_repeating_section + ): # if is_repeating_section, # its parent already initiliased an empty list # so we dont add it to our list of columns, # the repeating columns list will be @@ -550,11 +746,14 @@ def _update_ordered_columns_from_data(self, cursor): for (key, choices) in iteritems(self.select_multiples): # HACK to ensure choices are NOT duplicated if key in self.ordered_columns.keys(): - self.ordered_columns[key] = \ - remove_dups_from_list_maintain_order( - [choice.replace('/' + name, '/' + label) - if self.show_choice_labels else choice - for choice, name, label in choices]) + self.ordered_columns[key] = remove_dups_from_list_maintain_order( + [ + choice.replace("/" + name, "/" + label) + if self.show_choice_labels + else choice + for choice, name, label in choices + ] + ) # add ordered columns for gps fields for key in self.gps_fields: @@ -566,12 +765,17 @@ def _update_ordered_columns_from_data(self, cursor): # re index column repeats for (key, value) in iteritems(record): self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.dd, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) def _format_for_dataframe(self, cursor): """ @@ -581,10 +785,12 @@ def _format_for_dataframe(self, cursor): # split select multiples if self.split_select_multiples: record = self._split_select_multiples( - record, self.select_multiples, + record, + self.select_multiples, self.BINARY_SELECT_MULTIPLES, self.VALUE_SELECT_MULTIPLES, - show_choice_labels=self.show_choice_labels) + show_choice_labels=self.show_choice_labels, + ) # check for gps and split into # components i.e. latitude, longitude, # altitude, precision @@ -594,12 +800,17 @@ def _format_for_dataframe(self, cursor): # re index repeats for (key, value) in iteritems(record): reindexed = self._reindex( - key, value, self.ordered_columns, record, self.dd, + key, + value, + self.ordered_columns, + record, + self.dd, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, show_choice_labels=self.show_choice_labels, - language=self.language) + language=self.language, + ) flat_dict.update(reindexed) yield flat_dict @@ -608,8 +819,9 @@ def export_to(self, path, dataview=None): self._build_ordered_columns(self.dd.survey, self.ordered_columns) if dataview: - cursor = dataview.query_data(dataview, all_data=True, - filter_query=self.filter_query) + cursor = dataview.query_data( + dataview, all_data=True, filter_query=self.filter_query + ) if isinstance(cursor, QuerySet): cursor = cursor.iterator() @@ -617,11 +829,15 @@ def export_to(self, path, dataview=None): data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns) - if [c for c in dataview.columns if xpath.startswith(c)]] - )) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + if [c for c in dataview.columns if xpath.startswith(c)] + ] + ) + ) else: try: cursor = self._query_data(self.filter_query) @@ -637,29 +853,40 @@ def export_to(self, path, dataview=None): # Unpack xform columns and data data = self._format_for_dataframe(cursor) - columns = list(chain.from_iterable( - [[xpath] if cols is None else cols - for (xpath, cols) in iteritems(self.ordered_columns)])) + columns = list( + chain.from_iterable( + [ + [xpath] if cols is None else cols + for (xpath, cols) in iteritems(self.ordered_columns) + ] + ) + ) # add extra columns columns += [col for col in self.extra_columns] - for field in self.dd.get_survey_elements_of_type('osm'): - columns += OsmData.get_tag_keys(self.xform, - field.get_abbreviated_xpath(), - include_prefix=True) + for field in self.dd.get_survey_elements_of_type("osm"): + columns += OsmData.get_tag_keys( + self.xform, field.get_abbreviated_xpath(), include_prefix=True + ) columns_with_hxl = self.include_hxl and get_columns_with_hxl( - self.dd.survey_elements) - - write_to_csv(path, data, columns, - columns_with_hxl=columns_with_hxl, - remove_group_name=self.remove_group_name, - dd=self.dd, group_delimiter=self.group_delimiter, - include_labels=self.include_labels, - include_labels_only=self.include_labels_only, - include_hxl=self.include_hxl, - win_excel_utf8=self.win_excel_utf8, - total_records=self.total_records, - index_tags=self.index_tags, - language=self.language) + self.dd.survey_elements + ) + + write_to_csv( + path, + data, + columns, + columns_with_hxl=columns_with_hxl, + remove_group_name=self.remove_group_name, + dd=self.dd, + group_delimiter=self.group_delimiter, + include_labels=self.include_labels, + include_labels_only=self.include_labels_only, + include_hxl=self.include_hxl, + win_excel_utf8=self.win_excel_utf8, + total_records=self.total_records, + index_tags=self.index_tags, + language=self.language, + ) diff --git a/onadata/libs/utils/dict_tools.py b/onadata/libs/utils/dict_tools.py index f57c87c450..466ae6820f 100644 --- a/onadata/libs/utils/dict_tools.py +++ b/onadata/libs/utils/dict_tools.py @@ -4,8 +4,6 @@ """ import json -from past.builtins import basestring - def get_values_matching_key(doc, key): """ @@ -37,7 +35,7 @@ def list_to_dict(items, value): key = items.pop() result = {} - bracket_index = key.find('[') + bracket_index = key.find("[") if bracket_index > 0: value = [value] @@ -58,15 +56,17 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): for row in list_of_dicts: for k, v in row.items(): if isinstance(v, list): - z = merge_list_of_dicts(result[k] + v if k in result else v, - override_keys=override_keys) + z = merge_list_of_dicts( + result[k] + v if k in result else v, override_keys=override_keys + ) result[k] = z if isinstance(z, list) else [z] else: if k in result: if isinstance(v, dict): try: result[k] = merge_list_of_dicts( - [result[k], v], override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) except AttributeError as e: # If the key is within the override_keys # (Is a select_multiple question) We make @@ -74,12 +74,15 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): # more accurate as they usually mean that # the select_multiple has been split into # separate columns for each choice - if override_keys and isinstance(result[k], str)\ - and k in override_keys: + if ( + override_keys + and isinstance(result[k], str) + and k in override_keys + ): result[k] = {} result[k] = merge_list_of_dicts( - [result[k], v], - override_keys=override_keys) + [result[k], v], override_keys=override_keys + ) else: raise e else: @@ -95,11 +98,11 @@ def remove_indices_from_dict(obj): Removes indices from a obj dict. """ if not isinstance(obj, dict): - raise ValueError(u"Expecting a dict, found: {}".format(type(obj))) + raise ValueError("Expecting a dict, found: {}".format(type(obj))) result = {} for key, val in obj.items(): - bracket_index = key.find('[') + bracket_index = key.find("[") key = key[:bracket_index] if bracket_index > -1 else key val = remove_indices_from_dict(val) if isinstance(val, dict) else val if isinstance(val, list): @@ -126,7 +129,7 @@ def csv_dict_to_nested_dict(csv_dict, select_multiples=None): for key in list(csv_dict): result = {} value = csv_dict[key] - split_keys = key.split('/') + split_keys = key.split("/") if len(split_keys) == 1: result[key] = value @@ -147,8 +150,8 @@ def dict_lists2strings(adict): :param d: The dict to convert. :returns: The converted dict.""" for k, v in adict.items(): - if isinstance(v, list) and all([isinstance(e, basestring) for e in v]): - adict[k] = ' '.join(v) + if isinstance(v, list) and all([isinstance(e, str) for e in v]): + adict[k] = " ".join(v) elif isinstance(v, dict): adict[k] = dict_lists2strings(v) @@ -162,8 +165,8 @@ def dict_paths2dict(adict): result = {} for k, v in adict.items(): - if k.find('/') > 0: - parts = k.split('/') + if k.find("/") > 0: + parts = k.split("/") if len(parts) > 1: k = parts[0] for part in parts[1:]: @@ -181,7 +184,7 @@ def query_list_to_dict(query_list_str): data_list = json.loads(query_list_str) data_dict = dict() for value in data_list: - data_dict[value['label']] = value['text'] + data_dict[value["label"]] = value["text"] return data_dict @@ -190,7 +193,7 @@ def floip_response_headers_dict(data, xform_headers): """ Returns a dict from matching xform headers and floip responses. """ - headers = [i.split('/')[-1] for i in xform_headers] + headers = [i.split("/")[-1] for i in xform_headers] data = [i[4] for i in data] flow_dict = dict(zip(headers, data)) diff --git a/onadata/libs/utils/qrcode.py b/onadata/libs/utils/qrcode.py index 7fade1e99c..f82673b917 100644 --- a/onadata/libs/utils/qrcode.py +++ b/onadata/libs/utils/qrcode.py @@ -1,28 +1,41 @@ +# -*- coding: utf-8 -*- +""" +QR code utility function. +""" from base64 import b64encode from elaphe import barcode from io import BytesIO -from past.builtins import basestring -def generate_qrcode(message, stream=None, - eclevel='M', margin=10, - data_mode='8bits', format='PNG', scale=2.5): - """ Generate a QRCode, settings options and output.""" +def generate_qrcode( + message, + stream=None, + eclevel="M", + margin=10, + data_mode="8bits", + format="PNG", + scale=2.5, +): + """Generate a QRCode, settings options and output.""" if stream is None: stream = BytesIO() - if isinstance(message, basestring): + if isinstance(message, str): message = message.encode() - img = barcode('qrcode', message, - options=dict(version=9, eclevel=eclevel), - margin=margin, data_mode=data_mode, scale=scale) + img = barcode( + "qrcode", + message, + options=dict(version=9, eclevel=eclevel), + margin=margin, + data_mode=data_mode, + scale=scale, + ) img.save(stream, format) - datauri = "data:image/png;base64,%s"\ - % b64encode(stream.getvalue()).decode("utf-8") + datauri = "data:image/png;base64,%s" % b64encode(stream.getvalue()).decode("utf-8") stream.close() return datauri diff --git a/onadata/libs/utils/string.py b/onadata/libs/utils/string.py index 13d8f44105..1ece191bc0 100644 --- a/onadata/libs/utils/string.py +++ b/onadata/libs/utils/string.py @@ -1,6 +1,13 @@ -from past.builtins import basestring +# -*- coding: utf-8 -*- +""" +String utility function str2bool - converts yes, true, t, 1 to True +else returns the argument value v. +""" def str2bool(v): - return v.lower() in ( - 'yes', 'true', 't', '1') if isinstance(v, basestring) else v + """ + String utility function str2bool - converts "yes", "true", "t", "1" to True + else returns the argument value v. + """ + return v.lower() in ("yes", "true", "t", "1") if isinstance(v, str) else v diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 62db834666..deaec9556f 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -6,13 +6,12 @@ import sys import zipfile from builtins import open -from future.utils import iteritems from json.decoder import JSONDecodeError from tempfile import NamedTemporaryFile from typing import Dict from xml.dom import minidom -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings from django.core.files.storage import get_storage_class @@ -24,16 +23,14 @@ from onadata.libs.utils.common_tags import EXPORT_MIMES from onadata.libs.utils.common_tools import report_exception -SLASH = u"/" +SLASH = "/" def image_urls_for_form(xform): """Return image urls of all image attachments of the xform.""" return sum( - [ - image_urls(s) - for s in xform.instances.filter(deleted_at__isnull=True) - ], []) + [image_urls(s) for s in xform.instances.filter(deleted_at__isnull=True)], [] + ) def get_path(path, suffix): @@ -52,7 +49,7 @@ def image_urls(instance): """ default_storage = get_storage_class()() urls = [] - suffix = settings.THUMB_CONF['medium']['suffix'] + suffix = settings.THUMB_CONF["medium"]["suffix"] for attachment in instance.attachments.all(): path = get_path(attachment.media_file.name, suffix) if default_storage.exists(path): @@ -76,14 +73,15 @@ def parse_xform_instance(xml_str): # THIS IS OKAY FOR OUR USE CASE, BUT OTHER USERS SHOULD BEWARE. survey_data = dict(_path_value_pairs(root_node)) if len(list(_all_attributes(root_node))) != 1: - raise AssertionError(_( - u"There should be exactly one attribute in this document.")) - survey_data.update({ - common_tags.XFORM_ID_STRING: - root_node.getAttribute(u"id"), - common_tags.INSTANCE_DOC_NAME: - root_node.nodeName, - }) + raise AssertionError( + _("There should be exactly one attribute in this document.") + ) + survey_data.update( + { + common_tags.XFORM_ID_STRING: root_node.getAttribute("id"), + common_tags.INSTANCE_DOC_NAME: root_node.nodeName, + } + ) return survey_data @@ -105,8 +103,7 @@ def _path_value_pairs(node): if node.childNodes: # there's no data for this leaf node yield _path(node), None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node yield _path(node), node.childNodes[0].nodeValue else: @@ -130,7 +127,7 @@ def django_file(path, field_name, content_type): """Return an InMemoryUploadedFile object for file uploads.""" # adapted from here: http://groups.google.com/group/django-users/browse_th\ # read/thread/834f988876ff3c45/ - file_object = open(path, 'rb') + file_object = open(path, "rb") return InMemoryUploadedFile( file=file_object, @@ -138,7 +135,8 @@ def django_file(path, field_name, content_type): name=file_object.name, content_type=content_type, size=os.path.getsize(path), - charset=None) + charset=None, + ) def export_def_from_filename(filename): @@ -156,38 +154,38 @@ def get_client_ip(request): arguments: request -- HttpRequest object. """ - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: - return x_forwarded_for.split(',')[0] + return x_forwarded_for.split(",")[0] - return request.META.get('REMOTE_ADDR') + return request.META.get("REMOTE_ADDR") -def get_enketo_urls(form_url, - id_string, - instance_xml=None, - instance_id=None, - return_url=None, - **kwargs) -> Dict[str, str]: +def get_enketo_urls( + form_url, id_string, instance_xml=None, instance_id=None, return_url=None, **kwargs +) -> Dict[str, str]: """Return Enketo URLs.""" - if (not hasattr(settings, 'ENKETO_URL') or - not hasattr(settings, 'ENKETO_API_ALL_SURVEY_LINKS_PATH') or - not hasattr(settings, 'ENKETO_API_TOKEN') or - settings.ENKETO_API_TOKEN == ''): + if ( + not hasattr(settings, "ENKETO_URL") + or not hasattr(settings, "ENKETO_API_ALL_SURVEY_LINKS_PATH") + or not hasattr(settings, "ENKETO_API_TOKEN") + or settings.ENKETO_API_TOKEN == "" + ): return False - url = urljoin( - settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) + url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_ALL_SURVEY_LINKS_PATH) - values = {'form_id': id_string, 'server_url': form_url} + values = {"form_id": id_string, "server_url": form_url} if instance_id is not None and instance_xml is not None: url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_INSTANCE_PATH) - values.update({ - 'instance': instance_xml, - 'instance_id': instance_id, - # convert to unicode string in python3 compatible way - 'return_url': u'%s' % return_url - }) + values.update( + { + "instance": instance_xml, + "instance_id": instance_id, + # convert to unicode string in python3 compatible way + "return_url": "%s" % return_url, + } + ) if kwargs: # Kwargs need to take note of xform variable paths i.e. @@ -197,11 +195,15 @@ def get_enketo_urls(form_url, response = requests.post( url, data=values, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) resp_content = response.content - resp_content = resp_content.decode('utf-8') if hasattr( - resp_content, 'decode') else resp_content + resp_content = ( + resp_content.decode("utf-8") + if hasattr(resp_content, "decode") + else resp_content + ) if response.status_code in [200, 201]: try: data = json.loads(resp_content) @@ -219,16 +221,17 @@ def handle_enketo_error(response): try: data = json.loads(response.content) except (ValueError, JSONDecodeError): - report_exception("HTTP Error {}".format(response.status_code), - response.text, sys.exc_info()) + report_exception( + "HTTP Error {}".format(response.status_code), response.text, sys.exc_info() + ) if response.status_code == 502: raise EnketoError( - u"Sorry, we cannot load your form right now. Please try " - "again later.") + "Sorry, we cannot load your form right now. Please try " "again later." + ) raise EnketoError() else: - if 'message' in data: - raise EnketoError(data['message']) + if "message" in data: + raise EnketoError(data["message"]) raise EnketoError(response.text) @@ -237,7 +240,7 @@ def generate_enketo_form_defaults(xform, **kwargs): defaults = {} if kwargs: - for (name, value) in iteritems(kwargs): + for (name, value) in kwargs.items(): field = xform.get_survey_element(name) if field: defaults["defaults[{}]".format(field.get_xpath())] = value @@ -249,7 +252,7 @@ def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file tmp = NamedTemporaryFile() - with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name @@ -261,7 +264,8 @@ def create_attachments_zipfile(attachments): report_exception( "Create attachment zip exception", "File is greater than {} bytes".format( - settings.ZIP_REPORT_ATTACHMENT_LIMIT) + settings.ZIP_REPORT_ATTACHMENT_LIMIT + ), ) break else: @@ -281,8 +285,8 @@ def get_form(kwargs): from onadata.apps.logger.models import XForm from django.http import Http404 - queryset = kwargs.pop('queryset', XForm.objects.all()) - kwargs['deleted_at__isnull'] = True + queryset = kwargs.pop("queryset", XForm.objects.all()) + kwargs["deleted_at__isnull"] = True xform = queryset.filter(**kwargs).first() if xform: return xform @@ -290,12 +294,14 @@ def get_form(kwargs): raise Http404("XForm does not exist.") -def get_form_url(request, - username=None, - protocol='https', - preview=False, - xform_pk=None, - generate_consistent_urls=False): +def get_form_url( + request, + username=None, + protocol="https", + preview=False, + xform_pk=None, + generate_consistent_urls=False, +): """ Return a form list url endpoint to be used to make a request to Enketo. @@ -309,17 +315,18 @@ def get_form_url(request, http_host = settings.TEST_HTTP_HOST username = settings.TEST_USERNAME else: - http_host = request.META.get('HTTP_HOST', 'ona.io') + http_host = request.META.get("HTTP_HOST", "ona.io") - url = '%s://%s' % (protocol, http_host) + url = "%s://%s" % (protocol, http_host) if preview: - url += '/preview' + url += "/preview" if xform_pk and generate_consistent_urls: url += "/enketo/{}".format(xform_pk) elif username: - url += "/{}/{}".format(username, xform_pk) if xform_pk \ - else "/{}".format(username) + url += ( + "/{}/{}".format(username, xform_pk) if xform_pk else "/{}".format(username) + ) return url diff --git a/onadata/settings/common.py b/onadata/settings/common.py index fb7945a27c..62b03ff00b 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -19,19 +19,17 @@ from celery.signals import after_setup_logger from django.core.exceptions import SuspiciousOperation from django.utils.log import AdminEmailHandler -from past.builtins import basestring # setting default encoding to utf-8 -if sys.version[0] == '2': +if sys.version[0] == "2": reload(sys) sys.setdefaultencoding("utf-8") CURRENT_FILE = os.path.abspath(__file__) -PROJECT_ROOT = os.path.realpath( - os.path.join(os.path.dirname(CURRENT_FILE), '../')) +PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(CURRENT_FILE), "../")) PRINT_EXCEPTION = False -TEMPLATED_EMAIL_TEMPLATE_DIR = 'templated_email/' +TEMPLATED_EMAIL_TEMPLATE_DIR = "templated_email/" ADMINS = ( # ('Your Name', 'your_email@example.com'), @@ -39,9 +37,9 @@ MANAGERS = ADMINS -DEFAULT_FROM_EMAIL = 'noreply@ona.io' -SHARE_PROJECT_SUBJECT = '{} Ona Project has been shared with you.' -SHARE_ORG_SUBJECT = '{}, You have been added to {} organisation.' +DEFAULT_FROM_EMAIL = "noreply@ona.io" +SHARE_PROJECT_SUBJECT = "{} Ona Project has been shared with you." +SHARE_ORG_SUBJECT = "{}, You have been added to {} organisation." DEFAULT_SESSION_EXPIRY_TIME = 21600 # 6 hours DEFAULT_TEMP_TOKEN_EXPIRY_TIME = 21600 # 6 hours @@ -52,21 +50,21 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'America/New_York' +TIME_ZONE = "America/New_York" # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" LANGUAGES = ( - ('fr', u'Français'), - ('en', u'English'), - ('es', u'Español'), - ('it', u'Italiano'), - ('km', u'ភាសាខ្មែរ'), - ('ne', u'नेपाली'), - ('nl', u'Nederlands'), - ('zh', u'中文'), + ("fr", "Français"), + ("en", "English"), + ("es", "Español"), + ("it", "Italiano"), + ("km", "ភាសាខ្មែរ"), + ("ne", "नेपाली"), + ("nl", "Nederlands"), + ("zh", "中文"), ) SITE_ID = 1 @@ -82,39 +80,39 @@ # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = 'http://localhost:8000/media/' +MEDIA_URL = "http://localhost:8000/media/" # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') +STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") # URL prefix for static files. # Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Enketo URL -ENKETO_PROTOCOL = 'https' -ENKETO_URL = 'https://enketo.ona.io/' -ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' -ENKETO_API_INSTANCE_PATH = '/api_v2/instance' -ENKETO_API_TOKEN = '' +ENKETO_PROTOCOL = "https" +ENKETO_URL = "https://enketo.ona.io/" +ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" +ENKETO_API_INSTANCE_PATH = "/api_v2/instance" +ENKETO_API_TOKEN = "" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v2/instance/iframe" -ENKETO_API_SALT = 'secretsalt' +ENKETO_API_SALT = "secretsalt" VERIFY_SSL = True -ENKETO_AUTH_COOKIE = '__enketo' -ENKETO_META_UID_COOKIE = '__enketo_meta_uid' -ENKETO_META_USERNAME_COOKIE = '__enketo_meta_username' +ENKETO_AUTH_COOKIE = "__enketo" +ENKETO_META_UID_COOKIE = "__enketo_meta_uid" +ENKETO_META_USERNAME_COOKIE = "__enketo_meta_username" # Login URLs -LOGIN_URL = '/accounts/login/' -LOGIN_REDIRECT_URL = '/login_redirect/' +LOGIN_URL = "/accounts/login/" +LOGIN_REDIRECT_URL = "/login_redirect/" # URL prefix for admin static files -- CSS, JavaScript and images. # Make sure to use a trailing slash. # Examples: "http://foo.com/static/admin/", "/static/admin/". -ADMIN_MEDIA_PREFIX = '/static/admin/' +ADMIN_MEDIA_PREFIX = "/static/admin/" # Additional locations of static files STATICFILES_DIRS = ( @@ -126,29 +124,29 @@ # List of finder classes that know how to find static files in # various locations. STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(PROJECT_ROOT, 'libs/templates'), + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(PROJECT_ROOT, "libs/templates"), ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.template.context_processors.tz', - 'django.contrib.messages.context_processors.messages', - 'onadata.apps.main.context_processors.google_analytics', - 'onadata.apps.main.context_processors.site_name', + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages", + "onadata.apps.main.context_processors.google_analytics", + "onadata.apps.main.context_processors.site_name", ], }, }, @@ -156,80 +154,81 @@ MIDDLEWARE = ( - 'onadata.libs.profiling.sql.SqlTimingMiddleware', - 'django.middleware.http.ConditionalGetMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', + "onadata.libs.profiling.sql.SqlTimingMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks', - 'django.middleware.csrf.CsrfViewMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', - 'onadata.libs.utils.middleware.OperationalErrorMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks", + "django.middleware.csrf.CsrfViewMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", + "onadata.libs.utils.middleware.OperationalErrorMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ) -X_FRAME_OPTIONS = 'DENY' +X_FRAME_OPTIONS = "DENY" -LOCALE_PATHS = (os.path.join(PROJECT_ROOT, 'onadata.apps.main', 'locale'), ) +LOCALE_PATHS = (os.path.join(PROJECT_ROOT, "onadata.apps.main", "locale"),) -ROOT_URLCONF = 'onadata.apps.main.urls' +ROOT_URLCONF = "onadata.apps.main.urls" USE_TZ = True # needed by guardian -ANONYMOUS_DEFAULT_USERNAME = 'AnonymousUser' +ANONYMOUS_DEFAULT_USERNAME = "AnonymousUser" INSTALLED_APPS = ( - 'django.contrib.contenttypes', - 'django.contrib.auth', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.humanize', - 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.gis', - 'registration', - 'django_nose', - 'django_digest', - 'corsheaders', - 'oauth2_provider', - 'rest_framework', - 'rest_framework.authtoken', - 'taggit', - 'onadata.apps.logger', - 'onadata.apps.viewer', - 'onadata.apps.main', - 'onadata.apps.restservice', - 'onadata.apps.api', - 'guardian', - 'onadata.apps.sms_support', - 'onadata.libs', - 'reversion', - 'actstream', - 'onadata.apps.messaging.apps.MessagingConfig', - 'django_filters', - 'oidc', + "django.contrib.contenttypes", + "django.contrib.auth", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.humanize", + "django.contrib.admin", + "django.contrib.admindocs", + "django.contrib.gis", + "registration", + "django_nose", + "django_digest", + "corsheaders", + "oauth2_provider", + "rest_framework", + "rest_framework.authtoken", + "taggit", + "onadata.apps.logger", + "onadata.apps.viewer", + "onadata.apps.main", + "onadata.apps.restservice", + "onadata.apps.api", + "guardian", + "onadata.apps.sms_support", + "onadata.libs", + "reversion", + "actstream", + "onadata.apps.messaging.apps.MessagingConfig", + "django_filters", + "oidc", ) OAUTH2_PROVIDER = { # this is the list of available scopes - 'SCOPES': { - 'read': 'Read scope', - 'write': 'Write scope', - 'groups': 'Access to your groups'}, - 'OAUTH2_VALIDATOR_CLASS': 'onadata.libs.authentication.MasterReplicaOAuth2Validator' # noqa + "SCOPES": { + "read": "Read scope", + "write": "Write scope", + "groups": "Access to your groups", + }, + "OAUTH2_VALIDATOR_CLASS": "onadata.libs.authentication.MasterReplicaOAuth2Validator", # noqa } OPENID_CONNECT_VIEWSET_CONFIG = { "REDIRECT_AFTER_AUTH": "http://localhost:3000", "USE_SSO_COOKIE": True, "SSO_COOKIE_DATA": "email", - "JWT_SECRET_KEY": 'thesecretkey', - "JWT_ALGORITHM": 'HS256', + "JWT_SECRET_KEY": "thesecretkey", + "JWT_ALGORITHM": "HS256", "SSO_COOKIE_MAX_AGE": None, "SSO_COOKIE_DOMAIN": "localhost", "USE_AUTH_BACKEND": False, @@ -239,80 +238,71 @@ OPENID_CONNECT_AUTH_SERVERS = { "microsoft": { - "AUTHORIZATION_ENDPOINT": - "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", + "AUTHORIZATION_ENDPOINT": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", "CLIENT_ID": "client_id", - "JWKS_ENDPOINT": - "https://login.microsoftonline.com/common/discovery/v2.0/keys", + "JWKS_ENDPOINT": "https://login.microsoftonline.com/common/discovery/v2.0/keys", "SCOPE": "openid profile", - "TOKEN_ENDPOINT": - "https://login.microsoftonline.com/common/oauth2/v2.0/token", + "TOKEN_ENDPOINT": "https://login.microsoftonline.com/common/oauth2/v2.0/token", "END_SESSION_ENDPOINT": "http://localhost:3000", "REDIRECT_URI": "http://localhost:8000/oidc/msft/callback", "RESPONSE_TYPE": "id_token", "RESPONSE_MODE": "form_post", - "USE_NONCES": True + "USE_NONCES": True, } } REST_FRAMEWORK = { # Use hyperlinked styles by default. # Only used if the `serializer_class` attribute is not set on a view. - 'DEFAULT_MODEL_SERIALIZER_CLASS': - 'rest_framework.serializers.HyperlinkedModelSerializer', - + "DEFAULT_MODEL_SERIALIZER_CLASS": "rest_framework.serializers.HyperlinkedModelSerializer", # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.AllowAny', + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.AllowAny", ], - 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'onadata.libs.authentication.DigestAuthentication', - 'onadata.libs.authentication.TempTokenAuthentication', - 'onadata.libs.authentication.EnketoTokenAuthentication', - 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', - 'rest_framework.authentication.SessionAuthentication', - 'rest_framework.authentication.TokenAuthentication', + "DEFAULT_AUTHENTICATION_CLASSES": ( + "onadata.libs.authentication.DigestAuthentication", + "onadata.libs.authentication.TempTokenAuthentication", + "onadata.libs.authentication.EnketoTokenAuthentication", + "oauth2_provider.contrib.rest_framework.OAuth2Authentication", + "rest_framework.authentication.SessionAuthentication", + "rest_framework.authentication.TokenAuthentication", ), - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework_jsonp.renderers.JSONPRenderer', - 'rest_framework_csv.renderers.CSVRenderer', + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "rest_framework_jsonp.renderers.JSONPRenderer", + "rest_framework_csv.renderers.CSVRenderer", ), } SWAGGER_SETTINGS = { - "exclude_namespaces": [], # List URL namespaces to ignore - "api_version": '1.0', # Specify your API's version (optional) - "enabled_methods": [ # Methods to enable in UI - 'get', - 'post', - 'put', - 'patch', - 'delete' + "exclude_namespaces": [], # List URL namespaces to ignore + "api_version": "1.0", # Specify your API's version (optional) + "enabled_methods": [ # Methods to enable in UI + "get", + "post", + "put", + "patch", + "delete", ], } CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_CREDENTIALS = True -CORS_ORIGIN_WHITELIST = ( - 'http://dev.ona.io', -) -CORS_URLS_ALLOW_ALL_REGEX = ( - r'^/api/v1/osm/.*$', -) +CORS_ORIGIN_WHITELIST = ("http://dev.ona.io",) +CORS_URLS_ALLOW_ALL_REGEX = (r"^/api/v1/osm/.*$",) USE_THOUSAND_SEPARATOR = True COMPRESS = True # extra data stored with users -AUTH_PROFILE_MODULE = 'onadata.apps.main.UserProfile' +AUTH_PROFILE_MODULE = "onadata.apps.main.UserProfile" # case insensitive usernames AUTHENTICATION_BACKENDS = ( - 'onadata.apps.main.backends.ModelBackend', - 'guardian.backends.ObjectPermissionBackend', + "onadata.apps.main.backends.ModelBackend", + "guardian.backends.ObjectPermissionBackend", ) # Settings for Django Registration @@ -343,55 +333,49 @@ def skip_suspicious_operations(record): # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s' + - ' %(process)d %(thread)d %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "%(levelname)s %(asctime)s %(module)s" + + " %(process)d %(thread)d %(message)s" }, - 'profiler': { - 'format': '%(levelname)s %(asctime)s %(message)s' + "simple": {"format": "%(levelname)s %(message)s"}, + "profiler": {"format": "%(levelname)s %(asctime)s %(message)s"}, + "sql": { + "format": "%(levelname)s %(process)d %(thread)d" + + " %(time)s seconds %(message)s %(sql)s" }, - 'sql': { - 'format': '%(levelname)s %(process)d %(thread)d' + - ' %(time)s seconds %(message)s %(sql)s' + "sql_totals": { + "format": "%(levelname)s %(process)d %(thread)d %(time)s seconds" + + " %(message)s %(num_queries)s sql queries" }, - 'sql_totals': { - 'format': '%(levelname)s %(process)d %(thread)d %(time)s seconds' + - ' %(message)s %(num_queries)s sql queries' - } }, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - }, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, # Define filter for suspicious urls - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "skip_suspicious_operations": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_suspicious_operations, }, }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false', 'skip_suspicious_operations'], - 'class': 'django.utils.log.AdminEmailHandler' + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false", "skip_suspicious_operations"], + "class": "django.utils.log.AdminEmailHandler", }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'verbose', - 'stream': sys.stdout + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + "stream": sys.stdout, }, - 'audit': { - 'level': 'DEBUG', - 'class': 'onadata.libs.utils.log.AuditLogHandler', - 'formatter': 'verbose', - 'model': 'onadata.apps.main.models.audit.AuditLog' + "audit": { + "level": "DEBUG", + "class": "onadata.libs.utils.log.AuditLogHandler", + "formatter": "verbose", + "model": "onadata.apps.main.models.audit.AuditLog", }, # 'sql_handler': { # 'level': 'DEBUG', @@ -406,22 +390,18 @@ def skip_suspicious_operations(record): # 'stream': sys.stdout # } }, - 'loggers': { - 'django.request': { - 'handlers': ['mail_admins', 'console'], - 'level': 'DEBUG', - 'propagate': True, + "loggers": { + "django.request": { + "handlers": ["mail_admins", "console"], + "level": "DEBUG", + "propagate": True, }, - 'console_logger': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': True - }, - 'audit_logger': { - 'handlers': ['audit'], - 'level': 'DEBUG', - 'propagate': True + "console_logger": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": True, }, + "audit_logger": {"handlers": ["audit"], "level": "DEBUG", "propagate": True}, # 'sql_logger': { # 'handlers': ['sql_handler'], # 'level': 'DEBUG', @@ -432,12 +412,12 @@ def skip_suspicious_operations(record): # 'level': 'DEBUG', # 'propagate': True # } - } + }, } # PROFILE_API_ACTION_FUNCTION is used to toggle profiling a viewset's action PROFILE_API_ACTION_FUNCTION = False -PROFILE_LOG_BASE = '/tmp/' +PROFILE_LOG_BASE = "/tmp/" def configure_logging(logger, **kwargs): @@ -448,24 +428,24 @@ def configure_logging(logger, **kwargs): after_setup_logger.connect(configure_logging) -GOOGLE_STEP2_URI = 'http://ona.io/gwelcome' -GOOGLE_OAUTH2_CLIENT_ID = 'REPLACE ME' -GOOGLE_OAUTH2_CLIENT_SECRET = 'REPLACE ME' +GOOGLE_STEP2_URI = "http://ona.io/gwelcome" +GOOGLE_OAUTH2_CLIENT_ID = "REPLACE ME" +GOOGLE_OAUTH2_CLIENT_SECRET = "REPLACE ME" THUMB_CONF = { - 'large': {'size': 1280, 'suffix': '-large'}, - 'medium': {'size': 640, 'suffix': '-medium'}, - 'small': {'size': 240, 'suffix': '-small'}, + "large": {"size": 1280, "suffix": "-large"}, + "medium": {"size": 640, "suffix": "-medium"}, + "small": {"size": 240, "suffix": "-small"}, } # order of thumbnails from largest to smallest -THUMB_ORDER = ['large', 'medium', 'small'] -DEFAULT_IMG_FILE_TYPE = 'jpg' +THUMB_ORDER = ["large", "medium", "small"] +DEFAULT_IMG_FILE_TYPE = "jpg" # celery CELERY_TASK_ALWAYS_EAGER = False CELERY_TASK_IGNORE_RESULT = False CELERY_TASK_TRACK_STARTED = True -CELERY_IMPORTS = ('onadata.libs.utils.csv_import',) +CELERY_IMPORTS = ("onadata.libs.utils.csv_import",) CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD = 100000 # Bytes @@ -482,12 +462,12 @@ def configure_logging(logger, **kwargs): # default content length for submission requests DEFAULT_CONTENT_LENGTH = 10000000 -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' -NOSE_ARGS = ['--with-fixture-bundling', '--nologcapture', '--nocapture'] +TEST_RUNNER = "django_nose.NoseTestSuiteRunner" +NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] # fake endpoints for testing -TEST_HTTP_HOST = 'testserver.com' -TEST_USERNAME = 'bob' +TEST_HTTP_HOST = "testserver.com" +TEST_USERNAME = "bob" # specify the root folder which may contain a templates folder and a static # folder used to override templates for site specific details @@ -497,47 +477,46 @@ def configure_logging(logger, **kwargs): BINARY_SELECT_MULTIPLES = False # Use 'n/a' for empty values by default on csv exports -NA_REP = 'n/a' +NA_REP = "n/a" -if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, basestring): +if isinstance(TEMPLATE_OVERRIDE_ROOT_DIR, str): # site templates overrides - TEMPLATES[0]['DIRS'] = [ - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'templates'), - ] + TEMPLATES[0]['DIRS'] + TEMPLATES[0]["DIRS"] = [ + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "templates"), + ] + TEMPLATES[0]["DIRS"] # site static files path STATICFILES_DIRS += ( - os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, 'static'), + os.path.join(PROJECT_ROOT, TEMPLATE_OVERRIDE_ROOT_DIR, "static"), ) # Set wsgi url scheme to HTTPS -os.environ['wsgi.url_scheme'] = 'https' +os.environ["wsgi.url_scheme"] = "https" SUPPORTED_MEDIA_UPLOAD_TYPES = [ - 'audio/mp3', - 'audio/mpeg', - 'audio/wav', - 'audio/x-m4a', - 'image/jpeg', - 'image/png', - 'image/svg+xml', - 'text/csv', - 'text/json', - 'video/3gpp', - 'video/mp4', - 'application/json', - 'application/geo+json', - 'application/pdf', - 'application/msword', - 'application/vnd.ms-excel', - 'application/vnd.ms-powerpoint', - 'application/vnd.oasis.opendocument.text', - 'application/vnd.oasis.opendocument.spreadsheet', - 'application/vnd.oasis.opendocument.presentation', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.openxmlformats-officedocument.presentationml.\ - presentation', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/zip', + "audio/mp3", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", + "image/jpeg", + "image/png", + "image/svg+xml", + "text/csv", + "text/json", + "video/3gpp", + "video/mp4", + "application/json", + "application/pdf", + "application/msword", + "application/vnd.ms-excel", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.presentationml.\ + presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/zip", ] CSV_ROW_IMPORT_ASYNC_THRESHOLD = 100 @@ -547,27 +526,29 @@ def configure_logging(logger, **kwargs): PARSED_INSTANCE_DEFAULT_LIMIT = 1000000 PARSED_INSTANCE_DEFAULT_BATCHSIZE = 1000 -PROFILE_SERIALIZER = \ +PROFILE_SERIALIZER = ( "onadata.libs.serializers.user_profile_serializer.UserProfileSerializer" -ORG_PROFILE_SERIALIZER = \ +) +ORG_PROFILE_SERIALIZER = ( "onadata.libs.serializers.organization_serializer.OrganizationSerializer" +) BASE_VIEWSET = "onadata.libs.baseviewset.DefaultBaseViewset" path = os.path.join(PROJECT_ROOT, "..", "extras", "reserved_accounts.txt") EXPORT_WITH_IMAGE_DEFAULT = True try: - with open(path, 'r') as f: + with open(path, "r") as f: RESERVED_USERNAMES = [line.rstrip() for line in f] except EnvironmentError: RESERVED_USERNAMES = [] -STATIC_DOC = '/static/docs/index.html' +STATIC_DOC = "/static/docs/index.html" try: HOSTNAME = socket.gethostname() except Exception: - HOSTNAME = 'localhost' + HOSTNAME = "localhost" CACHE_MIXIN_SECONDS = 60 @@ -587,17 +568,17 @@ def configure_logging(logger, **kwargs): # email verification ENABLE_EMAIL_VERIFICATION = False -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -XLS_EXTENSIONS = ['xls', 'xlsx'] +XLS_EXTENSIONS = ["xls", "xlsx"] -CSV_EXTENSION = 'csv' +CSV_EXTENSION = "csv" PROJECT_QUERY_CHUNK_SIZE = 5000 # Prevents "The number of GET/POST parameters exceeded" exception DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000 -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # Time in minutes to lock out user from account LOCKOUT_TIME = 30 * 60 From ba6438091f891adc2550711f2a4301f5b2c1d2f5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 04:55:44 +0300 Subject: [PATCH 010/234] Switch to using six instead of future --- .../apps/api/models/organization_profile.py | 85 +- onadata/apps/api/models/team.py | 33 +- onadata/apps/api/models/temp_token.py | 10 +- .../tests/viewsets/test_project_viewset.py | 2231 ++++++++--------- .../viewsets/test_user_profile_viewset.py | 1294 +++++----- onadata/apps/api/tools.py | 297 ++- .../apps/api/viewsets/user_profile_viewset.py | 2 +- onadata/apps/api/viewsets/xform_viewset.py | 2 +- onadata/apps/logger/models/data_view.py | 296 ++- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/logger/models/open_data.py | 15 +- onadata/apps/logger/models/project.py | 149 +- onadata/apps/logger/models/survey_type.py | 5 +- onadata/apps/logger/models/xform.py | 647 ++--- .../logger/tests/test_briefcase_client.py | 89 +- onadata/apps/logger/xform_instance_parser.py | 82 +- onadata/apps/main/forms.py | 348 +-- onadata/apps/main/models/user_profile.py | 80 +- onadata/apps/main/tests/test_base.py | 276 +- .../apps/main/tests/test_form_enter_data.py | 2 +- onadata/apps/main/tests/test_process.py | 566 +++-- onadata/apps/main/tests/test_user_settings.py | 26 +- onadata/apps/restservice/models.py | 52 +- onadata/apps/restservice/services/textit.py | 20 +- onadata/apps/viewer/models/data_dictionary.py | 2 +- onadata/apps/viewer/models/export.py | 110 +- onadata/apps/viewer/parsed_instance_tools.py | 83 +- onadata/apps/viewer/tasks.py | 154 +- onadata/apps/viewer/tests/test_kml_export.py | 41 +- onadata/libs/renderers/renderers.py | 226 +- .../libs/serializers/attachment_serializer.py | 55 +- .../libs/serializers/metadata_serializer.py | 280 ++- .../serializers/password_reset_serializer.py | 93 +- .../libs/serializers/project_serializer.py | 360 +-- onadata/libs/serializers/widget_serializer.py | 89 +- onadata/libs/serializers/xform_serializer.py | 411 +-- onadata/libs/tests/utils/test_email.py | 125 +- onadata/libs/utils/briefcase_client.py | 144 +- onadata/libs/utils/csv_import.py | 2 +- onadata/libs/utils/decorators.py | 14 +- onadata/libs/utils/email.py | 68 +- onadata/libs/utils/export_builder.py | 1072 ++++---- onadata/libs/utils/export_tools.py | 511 ++-- onadata/libs/utils/gravatar.py | 14 +- onadata/libs/utils/osm.py | 109 +- onadata/settings/common.py | 2 + requirements/base.in | 14 +- requirements/base.pip | 117 +- setup.cfg | 2 +- 49 files changed, 5827 insertions(+), 4880 deletions(-) diff --git a/onadata/apps/api/models/organization_profile.py b/onadata/apps/api/models/organization_profile.py index fc58c95b81..50179b9208 100644 --- a/onadata/apps/api/models/organization_profile.py +++ b/onadata/apps/api/models/organization_profile.py @@ -6,7 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models.signals import post_delete, post_save -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -23,7 +23,7 @@ def org_profile_post_delete_callback(sender, instance, **kwargs): """ # delete the org_user too instance.user.delete() - safe_delete('{}{}'.format(IS_ORG, instance.pk)) + safe_delete("{}{}".format(IS_ORG, instance.pk)) def create_owner_team_and_assign_permissions(org): @@ -32,14 +32,13 @@ def create_owner_team_and_assign_permissions(org): assigns the group and user permissions """ team = Team.objects.create( - name=Team.OWNER_TEAM_NAME, organization=org.user, - created_by=org.created_by) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + name=Team.OWNER_TEAM_NAME, organization=org.user, created_by=org.created_by + ) + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") # pylint: disable=unpacking-non-sequence permission, _ = Permission.objects.get_or_create( - codename="is_org_owner", name="Organization Owner", - content_type=content_type) # pylint: disable= + codename="is_org_owner", name="Organization Owner", content_type=content_type + ) # pylint: disable= team.permissions.add(permission) org.creator.groups.add(team) @@ -53,23 +52,14 @@ def create_owner_team_and_assign_permissions(org): assign_perm(perm.codename, org.created_by, org) if org.userprofile_ptr: - for perm in get_perms_for_model( - org.userprofile_ptr.__class__): - assign_perm( - perm.codename, org.user, org.userprofile_ptr) + for perm in get_perms_for_model(org.userprofile_ptr.__class__): + assign_perm(perm.codename, org.user, org.userprofile_ptr) if org.creator: - assign_perm( - perm.codename, - org.creator, - org.userprofile_ptr) - - if org.created_by and\ - org.created_by != org.creator: - assign_perm( - perm.codename, - org.created_by, - org.userprofile_ptr) + assign_perm(perm.codename, org.creator, org.userprofile_ptr) + + if org.created_by and org.created_by != org.creator: + assign_perm(perm.codename, org.created_by, org.userprofile_ptr) return team @@ -88,20 +78,20 @@ class OrganizationProfile(UserProfile): """Organization: Extends the user profile for organization specific info - * What does this do? - - it has a createor - - it has owner(s), through permissions/group - - has members, through permissions/group - - no login access, no password? no registration like a normal user? - - created by a user who becomes the organization owner - * What relationships? + * What does this do? + - it has a createor + - it has owner(s), through permissions/group + - has members, through permissions/group + - no login access, no password? no registration like a normal user? + - created by a user who becomes the organization owner + * What relationships? """ class Meta: - app_label = 'api' + app_label = "api" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to an organization") + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to an organization"), ) is_organization = models.BooleanField(default=True) @@ -109,7 +99,7 @@ class Meta: creator = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return "%s[%s]" % (self.name, self.user.username) def save(self, *args, **kwargs): # pylint: disable=arguments-differ super(OrganizationProfile, self).save(*args, **kwargs) @@ -119,7 +109,7 @@ def remove_user_from_organization(self, user): :param user: The user to remove from this organization. """ - for group in user.groups.filter('%s#' % self.user.username): + for group in user.groups.filter("%s#" % self.user.username): user.groups.remove(group) def is_organization_owner(self, user): @@ -130,27 +120,32 @@ def is_organization_owner(self, user): :returns: Boolean whether user has organization level permissions. """ has_owner_group = user.groups.filter( - name='%s#%s' % (self.user.username, Team.OWNER_TEAM_NAME)) + name="%s#%s" % (self.user.username, Team.OWNER_TEAM_NAME) + ) return True if has_owner_group else False post_save.connect( - _post_save_create_owner_team, sender=OrganizationProfile, - dispatch_uid='create_owner_team_and_permissions') + _post_save_create_owner_team, + sender=OrganizationProfile, + dispatch_uid="create_owner_team_and_permissions", +) -post_delete.connect(org_profile_post_delete_callback, - sender=OrganizationProfile, - dispatch_uid='org_profile_post_delete_callback') +post_delete.connect( + org_profile_post_delete_callback, + sender=OrganizationProfile, + dispatch_uid="org_profile_post_delete_callback", +) # pylint: disable=model-no-explicit-unicode class OrgProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) class OrgProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" - content_object = models.ForeignKey( - OrganizationProfile, on_delete=models.CASCADE) + + content_object = models.ForeignKey(OrganizationProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index 0297fceb8b..4cb3f99c51 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -1,4 +1,4 @@ -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from django.db import models from django.db.models.signals import post_save @@ -16,24 +16,28 @@ class Team(Group): we should remove them from all teams and projects within the organization. """ + class Meta: - app_label = 'api' + app_label = "api" OWNER_TEAM_NAME = "Owners" organization = models.ForeignKey(User, on_delete=models.CASCADE) projects = models.ManyToManyField(Project) created_by = models.ForeignKey( - User, related_name='team_creator', null=True, blank=True, - on_delete=models.SET_NULL) + User, + related_name="team_creator", + null=True, + blank=True, + on_delete=models.SET_NULL, + ) - date_created = models.DateTimeField(auto_now_add=True, null=True, - blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) def __str__(self): # return a clear group name without username to user for viewing - return self.name.split('#')[1] + return self.name.split("#")[1] @property def team_name(self): @@ -42,8 +46,8 @@ def team_name(self): def save(self, *args, **kwargs): # allow use of same name in different organizations/users # concat with # - if not self.name.startswith('#'.join([self.organization.username])): - self.name = u'%s#%s' % (self.organization.username, self.name) + if not self.name.startswith("#".join([self.organization.username])): + self.name = "%s#%s" % (self.organization.username, self.name) super(Team, self).save(*args, **kwargs) @@ -55,12 +59,15 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if instance.created_by: assign_perm(perm.codename, instance.created_by, instance) - if hasattr(organization, 'creator') and \ - organization.creator != instance.created_by: + if ( + hasattr(organization, "creator") + and organization.creator != instance.created_by + ): assign_perm(perm.codename, organization.creator, instance) if organization.created_by != instance.created_by: assign_perm(perm.codename, organization.created_by, instance) -post_save.connect(set_object_permissions, sender=Team, - dispatch_uid='set_team_object_permissions') +post_save.connect( + set_object_permissions, sender=Team, dispatch_uid="set_team_object_permissions" +) diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index 0de4b6cc42..3f46fc53ae 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -7,9 +7,9 @@ from django.conf import settings from django.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible -AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') +AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") @python_2_unicode_compatible @@ -18,13 +18,15 @@ class TempToken(models.Model): """ The temporary authorization token model. """ + key = models.CharField(max_length=40, primary_key=True) user = models.OneToOneField( - AUTH_USER_MODEL, related_name='_user', on_delete=models.CASCADE) + AUTH_USER_MODEL, related_name="_user", on_delete=models.CASCADE + ) created = models.DateTimeField(auto_now_add=True) class Meta: - app_label = 'api' + app_label = "api" def save(self, *args, **kwargs): if not self.key: diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index f55ab99ed3..16c0da382d 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -5,7 +5,7 @@ import json import os from builtins import str -from future.utils import iteritems +from six import iteritems from operator import itemgetter from collections import OrderedDict @@ -20,11 +20,11 @@ import requests from onadata.apps.api import tools -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.team_viewset import TeamViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet @@ -33,29 +33,40 @@ from onadata.libs import permissions as role from onadata.libs.models.share_project import ShareProject from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key -from onadata.libs.permissions import (ROLES_ORDERED, DataEntryMinorRole, - DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, - OwnerRole, ReadOnlyRole, - ReadOnlyRoleNoDownload) -from onadata.libs.serializers.project_serializer import (BaseProjectSerializer, - ProjectSerializer) - -ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryRole, - EditorRole, - ManagerRole, - OwnerRole] - - -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +from onadata.libs.permissions import ( + ROLES_ORDERED, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRole, + ReadOnlyRoleNoDownload, +) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) + +ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, +] + + +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock(url, request): response = requests.Response() response.status_code = 201 - response._content = \ - '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n'\ - ' "code": "200"\n}' + response._content = ( + '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n' ' "code": "200"\n}' + ) return response @@ -65,38 +76,38 @@ def get_latest_tags(project): class TestProjectViewSet(TestAbstractViewSet): - def setUp(self): super(TestProjectViewSet, self).setUp() - self.view = ProjectViewSet.as_view({ - 'get': 'list', - 'post': 'create' - }) + self.view = ProjectViewSet.as_view({"get": "list", "post": "create"}) def tearDown(self): cache.clear() super(TestProjectViewSet, self).tearDown() - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.urlopen") def test_publish_xlsform_using_url_upload(self, mock_urlopen): with HTTMock(enketo_mock): self._project_create() - view = ProjectViewSet.as_view({ - 'post': 'forms' - }) + view = ProjectViewSet.as_view({"post": "forms"}) pre_count = XForm.objects.count() project_id = self.project.pk - xls_url = 'https://ona.io/examples/forms/tutorial/form.xlsx' + xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation_different_id_string.xlsx") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation_different_id_string.xlsx", + ) + + xls_file = open(path, "rb") mock_urlopen.return_value = xls_file - post_data = {'xls_url': xls_url} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) mock_urlopen.assert_called_with(xls_url) @@ -105,61 +116,63 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): self.assertEqual(XForm.objects.count(), pre_count + 1) self.assertEqual( XFormVersion.objects.filter( - xform__pk=response.data.get('formid')).count(), 1) + xform__pk=response.data.get("formid") + ).count(), + 1, + ) def test_projects_list(self): self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) def test_project_list_returns_projects_for_active_users_only(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice_user = alice_profile.user - shared_project = Project(name='demo2', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=alice_user, - organization=alice_user) + shared_project = Project( + name="demo2", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=alice_user, + organization=alice_user, + ) shared_project.save() # share project with self.user - shareProject = ShareProject( - shared_project, self.user.username, 'manager') + shareProject = ShareProject(shared_project, self.user.username, "manager") shareProject.save() # ensure when alice_user isn't active we can NOT # see the project she shared alice_user.is_active = False alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 1) - self.assertNotEqual(response.data[0].get( - 'projectid'), shared_project.id) + self.assertNotEqual(response.data[0].get("projectid"), shared_project.id) # ensure when alice_user is active we can # see the project she shared alice_user.is_active = True alice_user.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = self.view(request) self.assertEqual(len(response.data), 2) shared_project_in_response = False for project in response.data: - if project.get('projectid') == shared_project.id: + if project.get("projectid") == shared_project.id: shared_project_in_response = True break self.assertTrue(shared_project_in_response) @@ -170,66 +183,58 @@ def test_project_list_returns_users_own_project_is_shared_to(self): contains all the users the project has been shared too """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - share_project = ShareProject( - self.project, 'alice', 'manager' - ) + share_project = ShareProject(self.project, "alice", "manager") share_project.save() # Ensure alice is in the list of users # When an owner requests for the project data - req = self.factory.get('/', **self.extra) + req = self.factory.get("/", **self.extra) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 2) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 2) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) # Ensure project managers can view all users the project was shared to - davis_data = {'username': 'davis', 'email': 'davis@localhost.com'} + davis_data = {"username": "davis", "email": "davis@localhost.com"} davis_profile = self._create_user_profile(davis_data) - dave_extras = { - 'HTTP_AUTHORIZATION': f'Token {davis_profile.user.auth_token}' - } + dave_extras = {"HTTP_AUTHORIZATION": f"Token {davis_profile.user.auth_token}"} share_project = ShareProject( - self.project, davis_profile.user.username, 'manager' + self.project, davis_profile.user.username, "manager" ) share_project.save() - req = self.factory.get('/', **dave_extras) + req = self.factory.get("/", **dave_extras) resp = self.view(req) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.data[0]['users']), 3) - shared_users = [user['user'] for user in resp.data[0]['users']] + self.assertEqual(len(resp.data[0]["users"]), 3) + shared_users = [user["user"] for user in resp.data[0]["users"]] self.assertIn(alice_profile.user.username, shared_users) self.assertIn(self.user.username, shared_users) def test_projects_get(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - user_props = ['user', 'first_name', 'last_name', 'role', - 'is_org', 'metadata'] + user_props = ["user", "first_name", "last_name", "role", "is_org", "metadata"] user_props.sort() - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) # test serialized data - serializer = ProjectSerializer(self.project, - context={'request': request}) + serializer = ProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, serializer.data) self.assertIsNotNone(self.project_data) self.assertEqual(response.data, self.project_data) - res_user_props = list(response.data['users'][0]) + res_user_props = list(response.data["users"][0]) res_user_props.sort() self.assertEqual(res_user_props, user_props) @@ -240,105 +245,109 @@ def test_project_get_deleted_form(self): self.xform.deleted_at = self.xform.date_created self.xform.save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEqual(len(response.data.get('forms')), 0) + self.assertEqual(len(response.data.get("forms")), 0) self.assertEqual(response.status_code, 200) def test_none_empty_forms_and_dataview_properties_in_returned_json(self): self._publish_xls_form_to_project() self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertGreater(len(response.data.get('forms')), 0) - self.assertGreater( - len(response.data.get('data_views')), 0) - - form_obj_keys = list(response.data.get('forms')[0]) - data_view_obj_keys = list(response.data.get('data_views')[0]) - self.assertEqual(['date_created', - 'downloadable', - 'encrypted', - 'formid', - 'id_string', - 'is_merged_dataset', - 'last_submission_time', - 'last_updated_at', - 'name', - 'num_of_submissions', - 'published_by_formbuilder', - 'url'], - sorted(form_obj_keys)) - self.assertEqual(['columns', - 'dataviewid', - 'date_created', - 'date_modified', - 'instances_with_geopoints', - 'matches_parent', - 'name', - 'project', - 'query', - 'url', - 'xform'], - sorted(data_view_obj_keys)) + self.assertGreater(len(response.data.get("forms")), 0) + self.assertGreater(len(response.data.get("data_views")), 0) + + form_obj_keys = list(response.data.get("forms")[0]) + data_view_obj_keys = list(response.data.get("data_views")[0]) + self.assertEqual( + [ + "date_created", + "downloadable", + "encrypted", + "formid", + "id_string", + "is_merged_dataset", + "last_submission_time", + "last_updated_at", + "name", + "num_of_submissions", + "published_by_formbuilder", + "url", + ], + sorted(form_obj_keys), + ) + self.assertEqual( + [ + "columns", + "dataviewid", + "date_created", + "date_modified", + "instances_with_geopoints", + "matches_parent", + "name", + "project", + "query", + "url", + "xform", + ], + sorted(data_view_obj_keys), + ) self.assertEqual(response.status_code, 200) def test_projects_tags(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) - list_view = ProjectViewSet.as_view({ - 'get': 'list', - }) + view = ProjectViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) + list_view = ProjectViewSet.as_view( + { + "get": "list", + } + ) project_id = self.project.pk # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, [u'hello']) - self.assertEqual(get_latest_tags(self.project), [u'hello']) + self.assertEqual(response.data, ["hello"]) + self.assertEqual(get_latest_tags(self.project), ["hello"]) # check filter by tag - request = self.factory.get('/', data={"tags": "hello"}, **self.extra) + request = self.factory.get("/", data={"tags": "hello"}, **self.extra) self.project.refresh_from_db() request.user = self.user self.project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data[0], self.project_data) - request = self.factory.get('/', data={"tags": "goodbye"}, **self.extra) + request = self.factory.get("/", data={"tags": "goodbye"}, **self.extra) response = list_view(request, pk=project_id) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # remove tag "hello" - request = self.factory.delete('/', **self.extra) - response = view(request, pk=project_id, label='hello') + request = self.factory.delete("/", **self.extra) + response = view(request, pk=project_id, label="hello") self.assertEqual(response.status_code, 200) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) self.assertEqual(get_latest_tags(self.project), []) @@ -358,7 +367,7 @@ def test_project_create_other_account(self): # pylint: disable=C0103 Test that a user cannot create a project in a different user account without the right permission. """ - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) bob = self.user self._login_user_and_profile(alice_data) @@ -368,17 +377,23 @@ def test_project_create_other_account(self): # pylint: disable=C0103 } # Alice should not be able to create the form in bobs account. - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, { - 'owner': [u'You do not have permission to create a project in ' - 'the organization {}.'.format(bob)]}) + self.assertEqual( + response.data, + { + "owner": [ + "You do not have permission to create a project in " + "the organization {}.".format(bob) + ] + }, + ) self.assertEqual(Project.objects.count(), 0) # Give Alice the permission to create a project in Bob's account. ManagerRole.add(alice_profile.user, bob.profile) - request = self.factory.post('/projects', data=data, **self.extra) + request = self.factory.post("/projects", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -397,25 +412,24 @@ def test_create_duplicate_project(self): """ # data to create project data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } # current number of projects count = Project.objects.count() # create project using data - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) after_count = Project.objects.count() @@ -423,28 +437,24 @@ def test_create_duplicate_project(self): # create another project using the same data request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, { - u'non_field_errors': - [u'The fields name, owner must make a unique set.'] - } + response.data, + {"non_field_errors": ["The fields name, owner must make a unique set."]}, ) final_count = Project.objects.count() self.assertEqual(after_count, final_count) def test_projects_create_no_metadata(self): data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "public": False, } - self._project_create(project_data=data, - merge=False) + self._project_create(project_data=data, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -457,7 +467,7 @@ def test_projects_create_no_metadata(self): def test_projects_create_many_users(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) self._project_create() projects = Project.objects.filter(created_by=self.user) @@ -469,235 +479,234 @@ def test_projects_create_many_users(self): def test_publish_xls_form_to_project(self): self._publish_xls_form_to_project() - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self._publish_xls_form_to_project() def test_num_datasets(self): self._publish_xls_form_to_project() self.project.refresh_from_db() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - self.assertEqual(self.project_data['num_datasets'], 1) + self.project, context={"request": request} + ).data + self.assertEqual(self.project_data["num_datasets"], 1) def test_last_submission_date(self): self._publish_xls_form_to_project() self._make_submissions() - request = self.factory.post('/', data={}, **self.extra) + request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project.refresh_from_db() self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - date_created = self.xform.instances.order_by( - '-date_created').values_list('date_created', flat=True)[0] - self.assertEqual(str(self.project_data['last_submission_date']), - str(date_created)) + self.project, context={"request": request} + ).data + date_created = self.xform.instances.order_by("-date_created").values_list( + "date_created", flat=True + )[0] + self.assertEqual( + str(self.project_data["last_submission_date"]), str(date_created) + ) def test_view_xls_form(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "forms"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) resultset = MetaData.objects.filter( - Q(object_id=self.xform.pk), Q(data_type='enketo_url') | - Q(data_type='enketo_preview_url') | - Q(data_type='enketo_single_submit_url')) - url = resultset.get(data_type='enketo_url') - preview_url = resultset.get(data_type='enketo_preview_url') - single_submit_url = resultset.get( - data_type='enketo_single_submit_url') - form_metadata = sorted([ + Q(object_id=self.xform.pk), + Q(data_type="enketo_url") + | Q(data_type="enketo_preview_url") + | Q(data_type="enketo_single_submit_url"), + ) + url = resultset.get(data_type="enketo_url") + preview_url = resultset.get(data_type="enketo_preview_url") + single_submit_url = resultset.get(data_type="enketo_single_submit_url") + form_metadata = sorted( + [ OrderedDict( [ - ('id', url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/::YY8M'), - ('data_type', 'enketo_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' - % url.pk), - ('date_created', url.date_created)]), + ("id", url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/::YY8M"), + ("data_type", "enketo_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ("url", "http://testserver/api/v1/metadata/%s" % url.pk), + ("date_created", url.date_created), + ] + ), OrderedDict( [ - ('id', preview_url.pk), - ('xform', self.xform.pk), - ('data_value', 'https://enketo.ona.io/preview/::YY8M'), - ('data_type', 'enketo_preview_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - preview_url.pk), - ('date_created', preview_url.date_created)]), + ("id", preview_url.pk), + ("xform", self.xform.pk), + ("data_value", "https://enketo.ona.io/preview/::YY8M"), + ("data_type", "enketo_preview_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + "http://testserver/api/v1/metadata/%s" % preview_url.pk, + ), + ("date_created", preview_url.date_created), + ] + ), OrderedDict( [ - ('id', single_submit_url.pk), - ('xform', self.xform.pk), - ('data_value', - 'http://enketo.ona.io/single/::XZqoZ94y'), - ('data_type', 'enketo_single_submit_url'), - ('data_file', None), - ('data_file_type', None), - ('media_url', None), - ('file_hash', None), - ('url', 'http://testserver/api/v1/metadata/%s' % - single_submit_url.pk), - ('date_created', single_submit_url.date_created)])], - key=itemgetter('id')) + ("id", single_submit_url.pk), + ("xform", self.xform.pk), + ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), + ("data_type", "enketo_single_submit_url"), + ("data_file", None), + ("data_file_type", None), + ("media_url", None), + ("file_hash", None), + ( + "url", + "http://testserver/api/v1/metadata/%s" + % single_submit_url.pk, + ), + ("date_created", single_submit_url.date_created), + ] + ), + ], + key=itemgetter("id"), + ) # test metadata content separately response_metadata = sorted( [dict(item) for item in response.data[0].pop("metadata")], - key=itemgetter('id')) + key=itemgetter("id"), + ) self.assertEqual(response_metadata, form_metadata) # remove metadata and date_modified - self.form_data.pop('metadata') - self.form_data.pop('date_modified') - self.form_data.pop('last_updated_at') - response.data[0].pop('date_modified') - response.data[0].pop('last_updated_at') - self.form_data.pop('has_id_string_changed') + self.form_data.pop("metadata") + self.form_data.pop("date_modified") + self.form_data.pop("last_updated_at") + response.data[0].pop("date_modified") + response.data[0].pop("last_updated_at") + self.form_data.pop("has_id_string_changed") self.assertDictEqual(dict(response.data[0]), dict(self.form_data)) def test_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertIn('forms', list(response.data)) - old_project_form_count = len(response.data['forms']) - old_project_num_datasets = response.data['num_datasets'] + self.assertIn("forms", list(response.data)) + old_project_form_count = len(response.data["forms"]) + old_project_num_datasets = response.data["num_datasets"] project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual(response.data['num_datasets'], 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["num_datasets"], 1) # Check if form transferred doesn't appear in the old project # details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=old_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual( - len(response.data['forms']), old_project_form_count - 1) - self.assertEqual( - response.data['num_datasets'], old_project_num_datasets - 1) + self.assertEqual(len(response.data["forms"]), old_project_form_count - 1) + self.assertEqual(response.data["num_datasets"], old_project_num_datasets - 1) def test_project_manager_can_assign_form_to_project(self): - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user as manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk old_project = self.project - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) - ShareProject(self.project, 'alice', 'manager').save() - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + ShareProject(self.project, "alice", "manager").save() + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.assertTrue(self.project.xform_set.filter(pk=self.xform.pk)) self.assertFalse(old_project.xform_set.filter(pk=self.xform.pk)) # check if form added appears in the project details - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIn('forms', list(response.data)) - self.assertEqual(len(response.data['forms']), 1) + self.assertIn("forms", list(response.data)) + self.assertEqual(len(response.data["forms"]), 1) def test_project_manager_can_assign_form_to_project_no_perm(self): # user must have owner/manager permissions - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() # alice user is not manager to both projects - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - self.assertFalse(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, self.project)) formid = self.xform.pk - project_name = u'another project' - self._project_create({'name': project_name}) + project_name = "another project" + self._project_create({"name": project_name}) self.assertTrue(self.project.name == project_name) ManagerRole.add(alice_profile.user, self.project) - self.assertTrue(ManagerRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) self._login_user_and_profile(alice_data) project_id = self.project.pk - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 403) def test_project_users_get_readonly_role_on_add_form(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) ReadOnlyRole.add(alice_profile.user, self.project) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) self._publish_xls_form_to_project() - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) - self.assertFalse(OwnerRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertFalse(OwnerRole.user_has_role(alice_profile.user, self.xform)) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_reject_form_transfer_if_target_account_has_id_string_already( - self, mock_send_mail): + self, mock_send_mail + ): # create bob's project and publish a form to it self._publish_xls_form_to_project() projectid = self.project.pk @@ -705,117 +714,119 @@ def test_reject_form_transfer_if_target_account_has_id_string_already( # create user alice alice_data = { - 'username': 'alice', - 'email': 'alice@localhost.com', - 'name': 'alice', - 'first_name': 'alice' + "username": "alice", + "email": "alice@localhost.com", + "name": "alice", + "first_name": "alice", } alice_profile = self._create_user_profile(alice_data) # share bob's project with alice - self.assertFalse( - ManagerRole.user_has_role(alice_profile.user, bobs_project)) - - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + self.assertFalse(ManagerRole.user_has_role(alice_profile.user, bobs_project)) + + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ManagerRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ManagerRole.user_has_role(alice_profile.user, self.xform)) # log in as alice self._login_user_and_profile(extra_post_data=alice_data) # publish a form to alice's project that shares an id_string with # form published by bob - publish_data = {'owner': 'http://testserver/api/v1/users/alice'} + publish_data = {"owner": "http://testserver/api/v1/users/alice"} self._publish_xls_form_to_project(publish_data=publish_data) alices_form = XForm.objects.filter( - user__username='alice', id_string='transportation_2011_07_25')[0] + user__username="alice", id_string="transportation_2011_07_25" + )[0] alices_project = alices_form.project bobs_form = XForm.objects.filter( - user__username='bob', id_string='transportation_2011_07_25')[0] + user__username="bob", id_string="transportation_2011_07_25" + )[0] formid = bobs_form.id # try transfering bob's form from bob's project to alice's project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=alices_project.id) self.assertEqual(response.status_code, 400) self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) # try transfering bob's form from to alice's other project with # no forms - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) new_project_id = self.project.id - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': formid} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": formid} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=new_project_id) self.assertEqual(response.status_code, 400) self.assertEquals( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_allow_form_transfer_if_org_is_owned_by_user( - self, mock_send_mail): + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_allow_form_transfer_if_org_is_owned_by_user(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() bobs_project = self.project - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # access bob's project initially to cache the forms list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) view(request, pk=bobs_project.pk) # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) org_project = self.project self.assertNotEqual(bobs_project.id, org_project.id) # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 201) # test that cached forms of a source project are cleared. Bob had one # forms initially and now it's been moved to the org project. - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=bobs_project.pk) bobs_results = response.data - self.assertListEqual(bobs_results.get('forms'), []) + self.assertListEqual(bobs_results.get("forms"), []) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create bob's project and publish a form to it self._publish_xls_form_to_project() @@ -823,60 +834,66 @@ def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create an organization with a project self._org_create() - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) # publish form to organization project self._publish_xls_form_to_project() # try transfering bob's form to an organization project he created - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.project.id) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('detail'), - u'Form with the same id_string already exists in this account') + response.data.get("detail"), + "Form with the same id_string already exists in this account", + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') - def test_form_transfer_when_org_creator_creates_project( - self, mock_send_mail): + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") + def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -884,11 +901,13 @@ def test_form_transfer_when_org_creator_creates_project( self.assertIn(alice_profile.user, owners_team.user_set.all()) # let bob create a project in org - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, user_bob) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -897,29 +916,32 @@ def test_form_transfer_when_org_creator_creates_project( self._login_user_and_profile(alice_data) self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) @@ -931,34 +953,31 @@ def test_project_transfer_upgrades_permissions(self): bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) alice_url = f'http://testserver/api/v1/users/{alice_data["username"]}' - self._project_create({ - 'name': 'test project', - 'owner': alice_url, - 'public': False, - }, merge=False) + self._project_create( + { + "name": "test project", + "owner": alice_url, + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin - self._login_user_and_profile( - {'username': bob.username, 'email': bob.email}) + self._login_user_and_profile({"username": bob.username, "email": bob.email}) self._org_create() self.assertEqual(self.organization.created_by, bob) - org_url = ( - 'http://testserver/api/v1/users/' - f'{self.organization.user.username}') - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + org_url = "http://testserver/api/v1/users/" f"{self.organization.user.username}" + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -966,44 +985,37 @@ def test_project_transfer_upgrades_permissions(self): self.assertIn(alice_profile.user, owners_team.user_set.all()) # Share project to bob as editor - data = {'username': bob.username, 'role': EditorRole.name} - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + data = {"username": bob.username, "role": EditorRole.name} + view = ProjectViewSet.as_view({"post": "share"}) alice_auth_token = Token.objects.get(user=alice_profile.user).key - auth_credentials = { - 'HTTP_AUTHORIZATION': f'Token {alice_auth_token}' - } - request = self.factory.post('/', data=data, **auth_credentials) + auth_credentials = {"HTTP_AUTHORIZATION": f"Token {alice_auth_token}"} + request = self.factory.post("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 204) # Transfer project to Bobs Organization - data = {'owner': org_url, 'name': alice_project.name} - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) - request = self.factory.patch('/', data=data, **auth_credentials) + data = {"owner": org_url, "name": alice_project.name} + view = ProjectViewSet.as_view({"patch": "partial_update"}) + request = self.factory.patch("/", data=data, **auth_credentials) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) # Ensure all Admins have admin privileges to the project # once transferred - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=alice_project.pk) self.assertEqual(response.status_code, 200) - project_users = response.data['users'] + project_users = response.data["users"] org_owners = get_or_create_organization_owners_team( - self.organization).user_set.all() + self.organization + ).user_set.all() for user in project_users: - owner = org_owners.filter(username=user['user']).first() + owner = org_owners.filter(username=user["user"]).first() if owner: - self.assertEqual(user['role'], OwnerRole.name) + self.assertEqual(user["role"], OwnerRole.name) @override_settings(ALLOW_PUBLIC_DATASETS=False) def test_disallow_public_project_creation(self): @@ -1011,55 +1023,55 @@ def test_disallow_public_project_creation(self): Test that an error is raised when a user tries to create a public project when public projects are disabled. """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'public': True + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "public": True, } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['public'][0], - "Public projects are currently disabled.") + response.data["public"][0], "Public projects are currently disabled." + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_form_transfer_when_org_admin_not_creator_creates_project( - self, mock_send_mail): + self, mock_send_mail + ): projects_count = Project.objects.count() xform_count = XForm.objects.count() user_bob = self.user # create user alice with a project - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'alice\'s project', - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': False, - }, merge=False) + self._project_create( + { + "name": "alice's project", + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": False, + }, + merge=False, + ) self.assertEqual(self.project.created_by, alice_profile.user) alice_project = self.project # create org owned by bob then make alice admin self._login_user_and_profile( - {'username': user_bob.username, 'email': user_bob.email}) + {"username": user_bob.username, "email": user_bob.email} + ) self._org_create() self.assertEqual(self.organization.created_by, user_bob) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': alice_profile.user.username, - 'role': OwnerRole.name} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) self.assertEqual(response.status_code, 201) @@ -1068,11 +1080,13 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a project in org self._login_user_and_profile(alice_data) - self._project_create({ - 'name': u'organization_project', - 'owner': 'http://testserver/api/v1/users/denoinc', - 'public': False, - }) + self._project_create( + { + "name": "organization_project", + "owner": "http://testserver/api/v1/users/denoinc", + "public": False, + } + ) self.assertEqual(self.project.created_by, alice_profile.user) org_project = self.project self.assertEqual(Project.objects.count(), projects_count + 2) @@ -1080,210 +1094,213 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a form in her personal project self.project = alice_project data = { - 'owner': ('http://testserver/api/v1/users/%s' - % alice_profile.user.username), - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "owner": ( + "http://testserver/api/v1/users/%s" % alice_profile.user.username + ), + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) self.assertEqual(self.xform.created_by, alice_profile.user) self.assertEqual(XForm.objects.count(), xform_count + 1) # let alice transfer the form to the organization project - view = ProjectViewSet.as_view({ - 'post': 'forms', - }) - post_data = {'formid': self.xform.id} - request = self.factory.post('/', data=post_data, **self.extra) + view = ProjectViewSet.as_view( + { + "post": "forms", + } + ) + post_data = {"formid": self.xform.id} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + role_class._remove_obj_permissions(alice_profile.user, self.project) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint_form_published_later(self, mock_send_mail): # create project self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk for role_class in ROLES: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) # publish form after project sharing self._publish_xls_form_to_project() - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.xform)) # Reset the mock called value to False mock_send_mail.called = False - data = {'username': 'alice', 'role': ''} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ""} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) - role_class._remove_obj_permissions(alice_profile.user, - self.project) + role_class._remove_obj_permissions(alice_profile.user, self.project) self.xform.delete() def test_project_share_remove_user(self): self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) projectid = self.project.pk role_class = ReadOnlyRole - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) def test_project_filter_by_owner(self): """ Test projects endpoint filter by owner. """ self._project_create() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com', - 'first_name': 'Alice', 'last_name': 'Alice'} + alice_data = { + "username": "alice", + "email": "alice@localhost.com", + "first_name": "Alice", + "last_name": "Alice", + } self._login_user_and_profile(alice_data) - ShareProject(self.project, self.user.username, 'readonly').save() + ShareProject(self.project, self.user.username, "readonly").save() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = view(request, pk=self.project.pk) request.user = self.user self.project.refresh_from_db() bobs_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - self._project_create({'name': 'another project'}) + self._project_create({"name": "another project"}) # both bob's and alice's projects - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data - result = [{'owner': p.get('owner'), - 'projectid': p.get('projectid')} for p in response.data] - bob_data = {'owner': 'http://testserver/api/v1/users/bob', - 'projectid': bobs_project_data.get('projectid')} - alice_data = {'owner': 'http://testserver/api/v1/users/alice', - 'projectid': alice_project_data.get('projectid')} + self.project, context={"request": request} + ).data + result = [ + {"owner": p.get("owner"), "projectid": p.get("projectid")} + for p in response.data + ] + bob_data = { + "owner": "http://testserver/api/v1/users/bob", + "projectid": bobs_project_data.get("projectid"), + } + alice_data = { + "owner": "http://testserver/api/v1/users/alice", + "projectid": alice_project_data.get("projectid"), + } self.assertIn(bob_data, result) self.assertIn(alice_data, result) # only bob's project - request = self.factory.get('/', {'owner': 'bob'}, **self.extra) + request = self.factory.get("/", {"owner": "bob"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(bobs_project_data, response.data) self.assertNotIn(alice_project_data, response.data) # only alice's project - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) self.assertIn(alice_project_data, response.data) # none existent user - request = self.factory.get('/', {'owner': 'noone'}, **self.extra) + request = self.factory.get("/", {"owner": "noone"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # authenticated user can view public project - joe_data = {'username': 'joe', 'email': 'joe@localhost.com'} + joe_data = {"username": "joe", "email": "joe@localhost.com"} self._login_user_and_profile(joe_data) # should not show private projects when filtered by owner - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertNotIn(bobs_project_data, response.data) @@ -1294,33 +1311,34 @@ def test_project_filter_by_owner(self): self.project.save() request.user = self.user alice_project_data = BaseProjectSerializer( - self.project, context={'request': request}).data + self.project, context={"request": request} + ).data - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertIn(alice_project_data, response.data) # should show deleted project public project when filtered by owner self.project.soft_delete() - request = self.factory.get('/', {'owner': 'alice'}, **self.extra) + request = self.factory.get("/", {"owner": "alice"}, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual([], response.data) def test_project_partial_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - metadata = '{"description": "Lorem ipsum",' \ - '"location": "Nakuru, Kenya",' \ - '"category": "water"' \ - '}' + metadata = ( + '{"description": "Lorem ipsum",' + '"location": "Nakuru, Kenya",' + '"category": "water"' + "}" + ) json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) @@ -1328,63 +1346,58 @@ def test_project_partial_updates(self): self.assertEqual(project.metadata, json_metadata) def test_cache_updated_on_project_update(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve', - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"get": "retrieve", "patch": "partial_update"}) self._project_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(False, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) projectid = self.project.pk - data = {'public': True} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": True} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(True, response.data.get("public")) - cached_project = cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}') + cached_project = cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}") self.assertEqual(cached_project, response.data) def test_project_put_updates(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'update' - }) + view = ProjectViewSet.as_view({"put": "update"}) projectid = self.project.pk data = { - 'name': u'updated name', - 'owner': 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'description', - 'location': 'Nairobi, Kenya', - 'category': 'health'} + "name": "updated name", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "description", + "location": "Nairobi, Kenya", + "category": "health", + }, } - data.update({'metadata': json.dumps(data.get('metadata'))}) - request = self.factory.put('/', data=data, **self.extra) + data.update({"metadata": json.dumps(data.get("metadata"))}) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=projectid) - data.update({'metadata': json.loads(data.get('metadata'))}) + data.update({"metadata": json.loads(data.get("metadata"))}) self.assertDictContainsSubset(data, response.data) def test_project_partial_updates_to_existing_metadata(self): self._project_create() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk metadata = '{"description": "Changed description"}' json_metadata = json.loads(metadata) - data = {'metadata': metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": metadata} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) project = Project.objects.get(pk=projectid) json_metadata.update(project.metadata) @@ -1393,15 +1406,14 @@ def test_project_partial_updates_to_existing_metadata(self): def test_project_update_shared_cascades_to_xforms(self): self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) projectid = self.project.pk - data = {'public': 'true'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"public": "true"} + request = self.factory.patch("/", data=data, **self.extra) response = view(request, pk=projectid) - xforms_status = XForm.objects.filter(project__pk=projectid)\ - .values_list('shared', flat=True) + xforms_status = XForm.objects.filter(project__pk=projectid).values_list( + "shared", flat=True + ) self.assertTrue(xforms_status[0]) self.assertEqual(response.status_code, 200) @@ -1409,15 +1421,13 @@ def test_project_add_star(self): self._project_create() self.assertEqual(len(self.project.user_stars.all()), 0) - view = ProjectViewSet.as_view({ - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(response.status_code, 204) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) @@ -1426,37 +1436,30 @@ def test_create_project_invalid_metadata(self): Make sure that invalid metadata values are outright rejected Test fix for: https://github.com/onaio/onadata/issues/977 """ - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': "null", - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": "null", + "public": False, } request = self.factory.post( - '/', - data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) def test_project_delete_star(self): self._project_create() - view = ProjectViewSet.as_view({ - 'delete': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"delete": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=self.project.pk) self.project.refresh_from_db() @@ -1467,70 +1470,67 @@ def test_project_get_starred_by(self): self._project_create() # add star as bob - view = ProjectViewSet.as_view({ - 'get': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"get": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # ensure email not shared user_profile_data = self.user_profile_data() - del user_profile_data['email'] - del user_profile_data['metadata'] + del user_profile_data["email"] + del user_profile_data["metadata"] - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) # add star as alice - request = self.factory.post('/', **self.extra) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # get star users as alice - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - alice_profile, bob_profile = sorted(response.data, - key=itemgetter('username')) - self.assertEquals(sorted(bob_profile.items()), - sorted(user_profile_data.items())) - self.assertEqual(alice_profile['username'], 'alice') + alice_profile, bob_profile = sorted(response.data, key=itemgetter("username")) + self.assertEquals( + sorted(bob_profile.items()), sorted(user_profile_data.items()) + ) + self.assertEqual(alice_profile["username"], "alice") def test_user_can_view_public_projects(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=public_project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['public'], True) - self.assertEqual(response.data['projectid'], public_project.pk) - self.assertEqual(response.data['name'], 'demo') + self.assertEqual(response.data["public"], True) + self.assertEqual(response.data["projectid"], public_project.pk) + self.assertEqual(response.data["name"], "demo") def test_projects_same_name_diff_case(self): data1 = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - self._project_create(project_data=data1, - merge=False) + self._project_create(project_data=data1, merge=False) self.assertIsNotNone(self.project) self.assertIsNotNone(self.project_data) @@ -1538,25 +1538,24 @@ def test_projects_same_name_diff_case(self): self.assertEqual(len(projects), 1) data2 = { - 'name': u'DEMO', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "DEMO", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) request = self.factory.post( - '/', data=json.dumps(data2), - content_type="application/json", **self.extra) + "/", data=json.dumps(data2), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) projects = Project.objects.all() self.assertEqual(len(projects), 1) @@ -1565,29 +1564,28 @@ def test_projects_same_name_diff_case(self): self.assertEqual(self.user, project.organization) def test_projects_get_exception(self): - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) # does not exists response = view(request, pk=11111) self.assertEqual(response.status_code, 404) - self.assertEqual(response.data, {u'detail': u'Not found.'}) + self.assertEqual(response.data, {"detail": "Not found."}) # invalid id - response = view(request, pk='1w') + response = view(request, pk="1w") self.assertEqual(response.status_code, 400) - error_msg = ("Invalid value for project_id. It must be a " - "positive integer.") - self.assertEqual(str(response.data['detail']), error_msg) + error_msg = "Invalid value for project_id. It must be a " "positive integer." + self.assertEqual(str(response.data["detail"]), error_msg) def test_publish_to_public_project(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project @@ -1597,9 +1595,13 @@ def test_publish_to_public_project(self): self.assertEquals(self.xform.shared_data, True) def test_public_form_private_project(self): - self.project = Project(name='demo', shared=False, - metadata=json.dumps({'description': ''}), - created_by=self.user, organization=self.user) + self.project = Project( + name="demo", + shared=False, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) self.project.save() self._publish_xls_form_to_project() @@ -1642,28 +1644,30 @@ def test_public_form_private_project(self): self.assertFalse(self.project.shared) def test_publish_to_public_project_public_form(self): - public_project = Project(name='demo', - shared=True, - metadata=json.dumps({'description': ''}), - created_by=self.user, - organization=self.user) + public_project = Project( + name="demo", + shared=True, + metadata=json.dumps({"description": ""}), + created_by=self.user, + organization=self.user, + ) public_project.save() self.project = public_project data = { - 'owner': 'http://testserver/api/v1/users/%s' + "owner": "http://testserver/api/v1/users/%s" % self.project.organization.username, - 'public': True, - 'public_data': True, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - 'bamboo_dataset': u'' + "public": True, + "public_data": True, + "description": "transportation_2011_07_25", + "downloadable": True, + "allows_sms": False, + "encrypted": False, + "sms_id_string": "transportation_2011_07_25", + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "bamboo_dataset": "", } self._publish_xls_form_to_project(publish_data=data, merge=False) @@ -1672,78 +1676,66 @@ def test_publish_to_public_project_public_form(self): def test_project_all_users_can_share_remove_themselves(self): self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() - self.assertTrue(role_class.user_has_role(self.user, - self.project)) - self.assertTrue(role_class.user_has_role(self.user, - self.xform)) - data['role'] = role_name + self.assertTrue(role_class.user_has_role(self.user, self.project)) + self.assertTrue(role_class.user_has_role(self.user, self.xform)) + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_owner_cannot_remove_self_if_no_other_owner(self): self._project_create() - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) ManagerRole.add(self.user, self.project) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} bob_profile = self._create_user_profile(tom_data) OwnerRole.add(bob_profile.user, self.project) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - error = {'remove': [u"Project requires at least one owner"]} + error = {"remove": ["Project requires at least one owner"]} self.assertEquals(response.data, error) - self.assertTrue(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertTrue(OwnerRole.user_has_role(bob_profile.user, self.project)) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} profile = self._create_user_profile(alice_data) OwnerRole.add(profile.user, self.project) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'tom', 'remove': True, 'role': 'owner'} + data = {"username": "tom", "remove": True, "role": "owner"} - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(OwnerRole.user_has_role(bob_profile.user, - self.project)) + self.assertFalse(OwnerRole.user_has_role(bob_profile.user, self.project)) def test_last_date_modified_changes_when_adding_new_form(self): self._project_create() @@ -1764,11 +1756,9 @@ def test_anon_project_form_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) + view = ProjectViewSet.as_view({"get": "forms"}) - request = self.factory.get('/') + request = self.factory.get("/") response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 404) @@ -1777,16 +1767,13 @@ def test_anon_project_list_endpoint(self): self._project_create() self._publish_xls_form_to_project() - view = ProjectViewSet.as_view({ - 'get': 'list' - }) + view = ProjectViewSet.as_view({"get": "list"}) self.project.shared = True self.project.save() - public_projects = Project.objects.filter( - shared=True).count() + public_projects = Project.objects.filter(shared=True).count() - request = self.factory.get('/') + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) @@ -1795,45 +1782,42 @@ def test_anon_project_list_endpoint(self): def test_project_manager_can_delete_xform(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(ManagerRole.user_has_role(alice, self.project)) - data = {'username': 'alice', 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(ManagerRole.user_has_role(alice, self.project)) - self.assertTrue(alice.has_perm('delete_xform', self.xform)) + self.assertTrue(alice.has_perm("delete_xform", self.xform)) def test_move_project_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk self.assertFalse(OwnerRole.user_has_role(alice, self.project)) - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) - data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % alice.username - } - request = self.factory.patch('/', data=data_patch, **self.extra) + data_patch = {"owner": "http://testserver/api/v1/users/%s" % alice.username} + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) # bob cannot move project if he does not have can_add_project project @@ -1842,7 +1826,7 @@ def test_move_project_owner(self): # Give bob permission. ManagerRole.add(self.user, alice_profile) - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() @@ -1853,51 +1837,48 @@ def test_cannot_share_project_to_owner(self): # create project and publish form to project self._publish_xls_form_to_project() - data = {'username': self.user.username, 'role': ManagerRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": self.user.username, + "role": ManagerRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['username'], [u"Cannot share project" - u" with the owner (bob)"]) + self.assertEqual( + response.data["username"], ["Cannot share project" " with the owner (bob)"] + ) self.assertTrue(OwnerRole.user_has_role(self.user, self.project)) def test_project_share_readonly(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) perms = role.get_object_users_with_permissions(self.project) for p in perms: - user = p.get('user') + user = p.get("user") if user == alice_profile.user: - r = p.get('role') + r = p.get("role") self.assertEquals(r, ReadOnlyRole.name) def test_move_project_owner_org(self): @@ -1907,19 +1888,17 @@ def test_move_project_owner_org(self): projectid = self.project.pk - view = ProjectViewSet.as_view({ - 'patch': 'partial_update' - }) + view = ProjectViewSet.as_view({"patch": "partial_update"}) old_org = self.project.organization data_patch = { - 'owner': 'http://testserver/api/v1/users/%s' % - self.organization.user.username + "owner": "http://testserver/api/v1/users/%s" + % self.organization.user.username } - request = self.factory.patch('/', data=data_patch, **self.extra) + request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) - for a in response.data.get('teams'): - self.assertIsNotNone(a.get('role')) + for a in response.data.get("teams"): + self.assertIsNotNone(a.get("role")) self.assertEqual(response.status_code, 200) project = Project.objects.get(pk=projectid) @@ -1929,7 +1908,7 @@ def test_move_project_owner_org(self): def test_project_share_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) # set the user inactive @@ -1939,266 +1918,242 @@ def test_project_share_inactive_user(self): projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 400) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) self.assertEqual( response.data, - {'username': [u'The following user(s) is/are not active: alice']}) + {"username": ["The following user(s) is/are not active: alice"]}, + ) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_remove_inactive_user(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # set the user inactive self.assertTrue(alice_profile.user.is_active) alice_profile.user.is_active = False alice_profile.user.save() - data = {'username': 'alice', 'role': ReadOnlyRole.name, "remove": True} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name, "remove": True} + request = self.factory.put("/", data=data, **self.extra) self.assertEqual(response.status_code, 204) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_project_share_readonly_no_downloads(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_data = self._create_user_profile(tom_data) projectid = self.project.pk self.assertFalse( - ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, - self.project)) + ReadOnlyRoleNoDownload.user_has_role(alice_profile.user, self.project) + ) - data = {'username': 'alice', 'role': ReadOnlyRoleNoDownload.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRoleNoDownload.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - data = {'username': 'tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') - elif user.get('user') == 'alice': - self.assertEquals(user.get('role'), 'readonly-no-download') - elif user.get('user') == 'tom': - self.assertEquals(user.get('role'), 'readonly') + if user.get("user") == "bob": + self.assertEquals(user.get("role"), "owner") + elif user.get("user") == "alice": + self.assertEquals(user.get("role"), "readonly-no-download") + elif user.get("user") == "tom": + self.assertEquals(user.get("role"), "readonly") def test_team_users_in_a_project(self): self._team_create() - project = Project.objects.create(name="Test Project", - organization=self.team.organization, - created_by=self.user, - metadata='{}') + project = Project.objects.create( + name="Test Project", + organization=self.team.organization, + created_by=self.user, + metadata="{}", + ) - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) user_chuck = chuck_profile.user - view = TeamViewSet.as_view({ - 'post': 'share'}) + view = TeamViewSet.as_view({"post": "share"}) - self.assertFalse(EditorRole.user_has_role(user_chuck, - project)) - data = {'role': EditorRole.name, - 'project': project.pk} + self.assertFalse(EditorRole.user_has_role(user_chuck, project)) + data = {"role": EditorRole.name, "project": project.pk} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, pk=self.team.pk) self.assertEqual(response.status_code, 204) tools.add_user_to_team(self.team, user_chuck) self.assertTrue(EditorRole.user_has_role(user_chuck, project)) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=project.pk) - self.assertIsNotNone(response.data['teams']) - self.assertEquals(3, len(response.data['teams'])) - self.assertEquals(response.data['teams'][2]['role'], 'editor') - self.assertEquals(response.data['teams'][2]['users'][0], - str(chuck_profile.user.username)) + self.assertIsNotNone(response.data["teams"]) + self.assertEquals(3, len(response.data["teams"])) + self.assertEquals(response.data["teams"][2]["role"], "editor") + self.assertEquals( + response.data["teams"][2]["users"][0], str(chuck_profile.user.username) + ) def test_project_accesible_by_admin_created_by_diff_admin(self): self._org_create() # user 1 - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) # user 2 - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - }) + view = OrganizationProfileViewSet.as_view( + { + "post": "members", + } + ) # save the org creator bob = self.user data = json.dumps( - {"username": alice_profile.user.username, - "role": OwnerRole.name}) + {"username": alice_profile.user.username, "role": OwnerRole.name} + ) # create admin 1 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(201, response.status_code) data = json.dumps( - {"username": chuck_profile.user.username, - "role": OwnerRole.name}) + {"username": chuck_profile.user.username, "role": OwnerRole.name} + ) # create admin 2 request = self.factory.post( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(201, response.status_code) # admin 2 creates a project self.user = chuck_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' % - self.organization.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" + % self.organization.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, } self._project_create(project_data=data) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # admin 1 tries to access project created by admin 2 self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEquals(200, response.status_code) # assert admin can add colaborators - tompoo_data = {'username': 'tompoo', 'email': 'tompoo@localhost.com'} + tompoo_data = {"username": "tompoo", "email": "tompoo@localhost.com"} self._create_user_profile(tompoo_data) - data = {'username': 'tompoo', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "tompoo", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share' - }) + view = ProjectViewSet.as_view({"put": "share"}) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) self.user = bob - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % bob.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % bob.auth_token} # remove from admin org data = json.dumps({"username": alice_profile.user.username}) - view = OrganizationProfileViewSet.as_view({ - 'delete': 'members' - }) + view = OrganizationProfileViewSet.as_view({"delete": "members"}) request = self.factory.delete( - '/', data=data, content_type='application/json', **self.extra) - response = view(request, user='denoinc') + "/", data=data, content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEquals(200, response.status_code) - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) self.user = alice_profile.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) @@ -2206,27 +2161,25 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): self.assertEquals(404, response.status_code) def test_public_project_on_creation(self): - view = ProjectViewSet.as_view({ - 'post': 'create' - }) + view = ProjectViewSet.as_view({"post": "create"}) data = { - 'name': u'demopublic', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': True + "name": "demopublic", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": True, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) - project = Project.prefetched.filter( - name=data['name'], created_by=self.user)[0] + project = Project.prefetched.filter(name=data["name"], created_by=self.user)[0] self.assertTrue(project.shared) @@ -2235,109 +2188,111 @@ def test_permission_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project columns = json.dumps(self.xform.get_field_name_xpaths_only()) - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': columns, - 'query': '[ ]'} + data = { + "name": "My DataView", + "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "columns": columns, + "query": "[ ]", + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertTrue(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_permission_not_passed_to_dataview_parent_form(self): self._project_create() project1 = self.project self._publish_xls_form_to_project() - data = {'name': u'demo2', - 'owner': - 'http://testserver/api/v1/users/%s' % self.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False} + data = { + "name": "demo2", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } self._project_create(data) project2 = self.project - data = {'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % project2.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]'} + data = { + "name": "My DataView", + "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]', + } self._create_dataview(data) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = ProjectViewSet.as_view({'put': 'share'}) + view = ProjectViewSet.as_view({"put": "share"}) - data = {'username': 'alice', 'remove': True} + data = {"username": "alice", "remove": True} for (role_name, role_class) in iteritems(role.ROLES): - ShareProject(self.project, 'alice', role_name).save() + ShareProject(self.project, "alice", role_name).save() self.assertFalse(role_class.user_has_role(self.user, project1)) self.assertTrue(role_class.user_has_role(self.user, project2)) self.assertFalse(role_class.user_has_role(self.user, self.xform)) - data['role'] = role_name + data["role"] = role_name - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertFalse(role_class.user_has_role(self.user, - project1)) - self.assertFalse(role_class.user_has_role(self.user, - self.project)) - self.assertFalse(role_class.user_has_role(self.user, - self.xform)) + self.assertFalse(role_class.user_has_role(self.user, project1)) + self.assertFalse(role_class.user_has_role(self.user, self.project)) + self.assertFalse(role_class.user_has_role(self.user, self.xform)) def test_project_share_xform_meta_perms(self): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk @@ -2346,59 +2301,56 @@ def test_project_share_xform_meta_perms(self): MetaData.xform_meta_permission(self.xform, data_value=data_value) for role_class in ROLES_ORDERED: - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice", "role": role_class.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertTrue(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertTrue(role_class.user_has_role(alice_profile.user, self.project)) if role_class in [EditorRole, EditorMinorRole]: self.assertFalse( - EditorRole.user_has_role(alice_profile.user, self.xform)) + EditorRole.user_has_role(alice_profile.user, self.xform) + ) self.assertTrue( - EditorMinorRole.user_has_role(alice_profile.user, - self.xform)) + EditorMinorRole.user_has_role(alice_profile.user, self.xform) + ) - elif role_class in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role_class in [DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole]: self.assertTrue( - DataEntryRole.user_has_role(alice_profile.user, - self.xform)) + DataEntryRole.user_has_role(alice_profile.user, self.xform) + ) else: self.assertTrue( - role_class.user_has_role(alice_profile.user, self.xform)) + role_class.user_has_role(alice_profile.user, self.xform) + ) - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_atomicity(self, mock_send_mail): # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) alice = alice_profile.user projectid = self.project.pk role_class = DataEntryOnlyRole - self.assertFalse(role_class.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': role_class.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice", + "role": role_class.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) @@ -2407,11 +2359,14 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(role_class.user_has_role(alice, self.xform)) - data['remove'] = True - request = self.factory.post('/', data=data, **self.extra) + data["remove"] = True + request = self.factory.post("/", data=data, **self.extra) mock_rm_xform_perms = MagicMock() - with patch('onadata.libs.models.share_project.remove_xform_permissions', mock_rm_xform_perms): # noqa + with patch( + "onadata.libs.models.share_project.remove_xform_permissions", + mock_rm_xform_perms, + ): # noqa mock_rm_xform_perms.side_effect = Exception() with self.assertRaises(Exception): response = view(request, pk=projectid) @@ -2420,7 +2375,7 @@ def test_project_share_atomicity(self, mock_send_mail): self.assertTrue(role_class.user_has_role(alice, self.project)) self.assertTrue(mock_rm_xform_perms.called) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) # permissions have changed for both project and xform @@ -2429,71 +2384,67 @@ def test_project_share_atomicity(self, mock_send_mail): def test_project_list_by_owner(self): # create project and publish form to project - sluggie_data = {'username': 'sluggie', - 'email': 'sluggie@localhost.com'} + sluggie_data = {"username": "sluggie", "email": "sluggie@localhost.com"} self._login_user_and_profile(sluggie_data) self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) projectid = self.project.pk - self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - data = {'username': 'alice', 'role': ReadOnlyRole.name} - request = self.factory.put('/', data=data, **self.extra) + data = {"username": "alice", "role": ReadOnlyRole.name} + request = self.factory.put("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'put': 'share', - 'get': 'list' - }) + view = ProjectViewSet.as_view({"put": "share", "get": "list"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - self.assertIsNone( - cache.get(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}'))) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}"))) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.project)) - self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, - self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) # Should list collaborators data = {"owner": "sluggie"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) - users = response.data[0]['users'] + users = response.data[0]["users"] self.assertEqual(response.status_code, 200) - self.assertIn({'first_name': u'Bob', 'last_name': u'erama', - 'is_org': False, 'role': 'readonly', 'user': u'alice', - 'metadata': {}}, users) + self.assertIn( + { + "first_name": "Bob", + "last_name": "erama", + "is_org": False, + "role": "readonly", + "user": "alice", + "metadata": {}, + }, + users, + ) def test_projects_soft_delete(self): self._project_create() - view = ProjectViewSet.as_view({ - 'get': 'list', - 'delete': 'destroy' - }) + view = ProjectViewSet.as_view({"get": "list", "delete": "destroy"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) project_id = self.project.pk - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - serializer = BaseProjectSerializer(self.project, - context={'request': request}) + serializer = BaseProjectSerializer(self.project, context={"request": request}) self.assertEqual(response.data, [serializer.data]) - self.assertIn('created_by', list(response.data[0])) + self.assertIn("created_by", list(response.data[0])) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.user = self.user response = view(request, pk=project_id) self.assertEqual(response.status_code, 204) @@ -2501,14 +2452,14 @@ def test_projects_soft_delete(self): self.project = Project.objects.get(pk=project_id) self.assertIsNotNone(self.project.deleted_at) - self.assertTrue('deleted-at' in self.project.name) + self.assertTrue("deleted-at" in self.project.name) self.assertEqual(self.project.deleted_by, self.user) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertFalse(serializer.data in response.data) @@ -2518,45 +2469,40 @@ def test_project_share_multiple_users(self): Test that the project can be shared to multiple users """ self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name} - request = self.factory.post('/', data=data, **self.extra) + data = {"username": "alice,tom", "role": ReadOnlyRole.name} + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "share", "get": "retrieve"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # get the users - users = response.data.get('users') + users = response.data.get("users") self.assertEqual(len(users), 3) for user in users: - if user.get('user') == 'bob': - self.assertEquals(user.get('role'), 'owner') + if user.get("user") == "bob": + self.assertEquals(user.get("role"), "owner") else: - self.assertEquals(user.get('role'), 'readonly') + self.assertEquals(user.get("role"), "readonly") - @patch('onadata.apps.api.viewsets.project_viewset.send_mail') + @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_sends_mail_on_multi_share(self, mock_send_mail): """ Test that on sharing a projects to multiple users mail is sent to all @@ -2564,109 +2510,96 @@ def test_sends_mail_on_multi_share(self, mock_send_mail): """ # create project and publish form to project self._publish_xls_form_to_project() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) - tom_data = {'username': 'tom', 'email': 'tom@localhost.com'} + tom_data = {"username": "tom", "email": "tom@localhost.com"} tom_profile = self._create_user_profile(tom_data) projectid = self.project.pk - self.assertFalse( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertFalse( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertFalse(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - data = {'username': 'alice,tom', 'role': ReadOnlyRole.name, - 'email_msg': 'I have shared the project with you'} - request = self.factory.post('/', data=data, **self.extra) + data = { + "username": "alice,tom", + "role": ReadOnlyRole.name, + "email_msg": "I have shared the project with you", + } + request = self.factory.post("/", data=data, **self.extra) - view = ProjectViewSet.as_view({ - 'post': 'share' - }) + view = ProjectViewSet.as_view({"post": "share"}) response = view(request, pk=projectid) self.assertEqual(response.status_code, 204) self.assertTrue(mock_send_mail.called) self.assertEqual(mock_send_mail.call_count, 2) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.project)) - self.assertTrue( - ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.project)) + self.assertTrue(ReadOnlyRole.user_has_role(tom_profile.user, self.xform)) def test_project_caching(self): """ Test project viewset caching always keeps the latest version of the project in cache """ - view = ProjectViewSet.as_view({ - 'post': 'forms', - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test on form detail update data returned from project viewset is # updated - form_view = XFormViewSet.as_view({ - 'patch': 'partial_update' - }) - post_data = {'title': 'new_name'} - request = self.factory.patch( - '/', data=post_data, **self.extra) + form_view = XFormViewSet.as_view({"patch": "partial_update"}) + post_data = {"title": "new_name"} + request = self.factory.patch("/", data=post_data, **self.extra) response = form_view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( - response.data['forms'][0]['last_submission_time'], - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"], + self.xform.time_of_last_submission(), + ) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) # Test that last_submission_time is updated correctly self._make_submissions() self.xform.refresh_from_db() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['forms']), 1) - self.assertEqual( - response.data['forms'][0]['name'], self.xform.title) - self.assertIsNotNone(response.data['forms'][0]['last_submission_time']) + self.assertEqual(len(response.data["forms"]), 1) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) + self.assertIsNotNone(response.data["forms"][0]["last_submission_time"]) returned_date = dateutil.parser.parse( - response.data['forms'][0]['last_submission_time']) - self.assertEqual( - returned_date, - self.xform.time_of_last_submission()) + response.data["forms"][0]["last_submission_time"] + ) + self.assertEqual(returned_date, self.xform.time_of_last_submission()) self.assertEqual( - response.data['forms'][0]['num_of_submissions'], - self.xform.num_of_submissions + response.data["forms"][0]["num_of_submissions"], + self.xform.num_of_submissions, ) - self.assertEqual(response.data['num_datasets'], 1) + self.assertEqual(response.data["num_datasets"], 1) diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index 9b1f8f7f89..b525b3f523 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -2,7 +2,7 @@ import json import os from builtins import str -from future.moves.urllib.parse import urlparse, parse_qs +from six.moves.urllib.parse import urlparse, parse_qs from django.contrib.auth.models import User from django.core.cache import cache @@ -18,47 +18,45 @@ from registration.models import RegistrationProfile from rest_framework.authtoken.models import Token -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.apps.main.models.user_profile import \ - set_kpi_formbuilder_permissions +from onadata.apps.main.models.user_profile import set_kpi_formbuilder_permissions from onadata.libs.authentication import DigestAuthentication -from onadata.libs.serializers.user_profile_serializer import \ - _get_first_last_names +from onadata.libs.serializers.user_profile_serializer import _get_first_last_names def _profile_data(): return { - 'username': u'deno', - 'first_name': u'Dennis', - 'last_name': u'erama', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, - 'name': u'Dennis erama' + "username": "deno", + "first_name": "Dennis", + "last_name": "erama", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, + "name": "Dennis erama", } class TestUserProfileViewSet(TestAbstractViewSet): - def setUp(self): super(self.__class__, self).setUp() - self.view = UserProfileViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'patch': 'partial_update', - 'put': 'update' - }) + self.view = UserProfileViewSet.as_view( + { + "get": "list", + "post": "create", + "patch": "partial_update", + "put": "update", + } + ) def tearDown(self): """ @@ -68,47 +66,52 @@ def tearDown(self): cache.clear() def test_profiles_list(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) data = self.user_profile_data() - del data['metadata'] + del data["metadata"] self.assertEqual(response.data, [data]) def test_user_profile_list(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertEqual( - sorted([dict(d) for d in response.data], key=lambda x: x['id']), - sorted([user_profile_data, deno_profile_data], - key=lambda x: x['id'])) + sorted([dict(d) for d in response.data], key=lambda x: x["id"]), + sorted([user_profile_data, deno_profile_data], key=lambda x: x["id"]), + ) self.assertEqual(len(response.data), 2) # Inactive user not in list @@ -117,36 +120,39 @@ def test_user_profile_list(self): response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertNotIn(user_deno.pk, [user['id'] for user in response.data]) + self.assertNotIn(user_deno.pk, [user["id"] for user in response.data]) def test_user_profile_list_with_and_without_users_param(self): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # anonymous user gets empty response - request = self.factory.get('/') + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) # authenicated user without users query param only gets his/her profile - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) user_profile_data = self.user_profile_data() - del user_profile_data['metadata'] + del user_profile_data["metadata"] self.assertDictEqual(user_profile_data, response.data[0]) # authenicated user with blank users query param only gets his/her # profile data = {"users": ""} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) @@ -155,128 +161,131 @@ def test_user_profile_list_with_and_without_users_param(self): # authenicated user with comma separated usernames as users query param # value gets profiles of the usernames provided data = {"users": "bob,deno"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request) deno_profile_data = _profile_data() - deno_profile_data.pop('password', None) - user_deno = User.objects.get(username='deno') - deno_profile_data.update({ - 'id': user_deno.pk, - 'url': 'http://testserver/api/v1/profiles/%s' % user_deno.username, - 'user': 'http://testserver/api/v1/users/%s' % user_deno.username, - 'gravatar': user_deno.profile.gravatar, - 'joined_on': user_deno.date_joined - }) + deno_profile_data.pop("password", None) + user_deno = User.objects.get(username="deno") + deno_profile_data.update( + { + "id": user_deno.pk, + "url": "http://testserver/api/v1/profiles/%s" % user_deno.username, + "user": "http://testserver/api/v1/users/%s" % user_deno.username, + "gravatar": user_deno.profile.gravatar, + "joined_on": user_deno.date_joined, + } + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) self.assertEqual( - [dict(i) for i in response.data], - [user_profile_data, deno_profile_data] + [dict(i) for i in response.data], [user_profile_data, deno_profile_data] ) def test_profiles_get(self): """Test get user profile""" - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) + response.data, {"detail": "Expected URL keyword argument `user`."} + ) # by username - response = view(request, user='bob') - self.assertNotEqual(response.get('Cache-Control'), None) + response = view(request, user="bob") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) # by username mixed case - response = view(request, user='BoB') + response = view(request, user="BoB") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, self.user_profile_data()) # by pk response = view(request, user=self.user.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.user_profile_data()) def test_profiles_get_anon(self): - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) - request = self.factory.get('/') - response = view(request, user='bob') + response.data, {"detail": "Expected URL keyword argument `user`."} + ) + request = self.factory.get("/") + response = view(request, user="bob") data = self.user_profile_data() - del data['email'] - del data['metadata'] + del data["email"] + del data["metadata"] self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profiles_get_org_anon(self): self._org_create() self.client.logout() - view = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') - response = view(request, user=self.company_data['org']) + view = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") + response = view(request, user=self.company_data["org"]) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], - self.company_data['name']) - self.assertIn('is_org', response.data) - self.assertEqual(response.data['is_org'], True) + self.assertEqual(response.data["first_name"], self.company_data["name"]) + self.assertIn("is_org", response.data) + self.assertEqual(response.data["is_org"], True) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_profile_create(self, mock_send_verification_email): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ('Dennis', 'erama') + password = data["password"] + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Dennis", "erama") self.assertEqual(response.data, data) self.assertTrue(mock_send_verification_email.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def _create_user_using_profiles_endpoint(self, data): request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -286,20 +295,19 @@ def test_return_204_if_email_verification_variables_are_not_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email', - 'post': 'send_verification_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') + view = UserProfileViewSet.as_view( + {"get": "verify_email", "post": "send_verification_email"} ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data, **self.extra) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data, **self.extra) response = view(request) self.assertEquals(response.status_code, 204) - data = {'username': data.get('username')} - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + data = {"username": data.get("username")} + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) self.assertEquals(response.status_code, 204) @@ -308,81 +316,72 @@ def test_verification_key_is_valid(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} - request = self.factory.get('/', data=_data) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username') - ) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEquals(response.data.get("username"), data.get("username")) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) def test_verification_key_is_valid_with_redirect_url_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) _data = { - 'verification_key': rp.activation_key, - 'redirect_url': 'http://red.ir.ect' + "verification_key": rp.activation_key, + "redirect_url": "http://red.ir.ect", } - request = self.factory.get('/', data=_data) + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 302) - self.assertIn('is_email_verified', response.url) - self.assertIn('username', response.url) + self.assertIn("is_email_verified", response.url) + self.assertIn("username", response.url) string_query_params = urlparse(response.url).query dict_query_params = parse_qs(string_query_params) - self.assertEquals(dict_query_params.get( - 'is_email_verified'), ['True']) - self.assertEquals( - dict_query_params.get('username'), - [data.get('username')] - ) + self.assertEquals(dict_query_params.get("is_email_verified"), ["True"]) + self.assertEquals(dict_query_params.get("username"), [data.get("username")]) - up = UserProfile.objects.get(user__username=data.get('username')) - self.assertIn('is_email_verified', up.metadata) - self.assertTrue(up.metadata.get('is_email_verified')) + up = UserProfile.objects.get(user__username=data.get("username")) + self.assertIn("is_email_verified", up.metadata) + self.assertTrue(up.metadata.get("is_email_verified")) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch( - ('onadata.apps.api.viewsets.user_profile_viewset.' - 'send_verification_email.delay') + ( + "onadata.apps.api.viewsets.user_profile_viewset." + "send_verification_email.delay" + ) ) - def test_sending_verification_email_succeeds( - self, mock_send_verification_email): + def test_sending_verification_email_succeeds(self, mock_send_verification_email): data = _profile_data() self._create_user_using_profiles_endpoint(data) - data = {'username': data.get('username')} - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + data = {"username": data.get("username")} + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) - user = User.objects.get(username=data.get('username')) - extra = {'HTTP_AUTHORIZATION': 'Token %s' % user.auth_token} - request = self.factory.post('/', data=data, **extra) + user = User.objects.get(username=data.get("username")) + extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} + request = self.factory.post("/", data=data, **extra) response = view(request) self.assertTrue(mock_send_verification_email.called) self.assertEquals(response.status_code, 200) self.assertEquals(response.data, "Verification email has been sent") - user = User.objects.get(username=data.get('username')) - self.assertFalse(user.profile.metadata.get('is_email_verified')) + user = User.objects.get(username=data.get("username")) + self.assertFalse(user.profile.metadata.get("is_email_verified")) @override_settings(VERIFIED_KEY_TEXT=None) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @@ -390,13 +389,11 @@ def test_sending_verification_email_fails(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'post': 'send_verification_email'}) + view = UserProfileViewSet.as_view({"post": "send_verification_email"}) # trigger permission error when username of requesting user is # different from username in post details - request = self.factory.post('/', - data={'username': 'None'}, - **self.extra) + request = self.factory.post("/", data={"username": "None"}, **self.extra) response = view(request) self.assertEquals(response.status_code, 403) @@ -406,28 +403,34 @@ def test_profile_require_auth(self): Test profile require_auth is True when REQUIRE_ODK_AUTHENTICATION is set to True. """ - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - self.assertTrue(response.data.get('require_auth')) + self.assertTrue(response.data.get("require_auth")) def test_profile_create_without_last_name(self): data = { - 'username': u'deno', - 'first_name': u'Dennis', - 'email': u'deno@columbia.edu', + "username": "deno", + "first_name": "Dennis", + "email": "deno@columbia.edu", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -436,127 +439,136 @@ def test_disallow_profile_create_w_same_username(self): self._create_user_using_profiles_endpoint(data) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertTrue( - 'deno already exists' in response.data['username'][0]) + self.assertTrue("deno already exists" in response.data["username"][0]) def test_profile_create_with_malfunctioned_email(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = { - 'username': u'nguyenquynh', - 'first_name': u'Nguy\u1ec5n Th\u1ecb', - 'last_name': u'Di\u1ec5m Qu\u1ef3nh', - 'email': u'onademo0+nguyenquynh@gmail.com\ufeff', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'nguyenquynh.com', - 'twitter': u'nguyenquynh', - 'require_auth': False, - 'password': u'onademo', - 'is_org': False, + "username": "nguyenquynh", + "first_name": "Nguy\u1ec5n Th\u1ecb", + "last_name": "Di\u1ec5m Qu\u1ef3nh", + "email": "onademo0+nguyenquynh@gmail.com\ufeff", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "nguyenquynh.com", + "twitter": "nguyenquynh", + "require_auth": False, + "password": "onademo", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - password = data['password'] - del data['password'] - - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/nguyenquynh' - data['user'] = 'http://testserver/api/v1/users/nguyenquynh' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined - data['name'] = "%s %s" % ( - u'Nguy\u1ec5n Th\u1ecb', u'Di\u1ec5m Qu\u1ef3nh') + password = data["password"] + del data["password"] + + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/nguyenquynh" + data["user"] = "http://testserver/api/v1/users/nguyenquynh" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined + data["name"] = "%s %s" % ("Nguy\u1ec5n Th\u1ecb", "Di\u1ec5m Qu\u1ef3nh") self.assertEqual(response.data, data) - user = User.objects.get(username='nguyenquynh') + user = User.objects.get(username="nguyenquynh") self.assertTrue(user.is_active) self.assertTrue(user.check_password(password), password) def test_profile_create_with_invalid_username(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - data['username'] = u'de' - del data['name'] + data["username"] = "de" + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('username'), - [u'Ensure this field has at least 3 characters.']) + response.data.get("username"), + ["Ensure this field has at least 3 characters."], + ) def test_profile_create_anon(self): data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json") + "/api/v1/profiles", data=json.dumps(data), content_type="application/json" + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - del data['email'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + del data["email"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - self.assertNotIn('email', response.data) + self.assertNotIn("email", response.data) def test_profile_create_missing_name_field(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['first_name'] - del data['name'] + del data["first_name"] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) response.render() - self.assertContains(response, - 'Either name or first_name should be provided', - status_code=400) + self.assertContains( + response, "Either name or first_name should be provided", status_code=400 + ) def test_split_long_name_to_first_name_and_last_name(self): - name = "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression "\ + name = ( + "(CPLTGL) Centre Pour la Promotion de la Liberte D'Expression " "et de la Tolerance Dans La Region de" + ) first_name, last_name = _get_first_last_names(name) self.assertEqual(first_name, "(CPLTGL) Centre Pour la Promot") self.assertEqual(last_name, "ion de la Liberte D'Expression") def test_partial_updates(self): - self.assertEqual(self.user.profile.country, u'US') - country = u'KE' - username = 'george' - metadata = {u'computer': u'mac'} + self.assertEqual(self.user.profile.country, "US") + country = "KE" + username = "george" + metadata = {"computer": "mac"} json_metadata = json.dumps(metadata) - data = {'username': username, - 'country': country, - 'metadata': json_metadata} - request = self.factory.patch('/', data=data, **self.extra) + data = {"username": username, "country": country, "metadata": json_metadata} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -568,13 +580,10 @@ def test_partial_updates_empty_metadata(self): profile = UserProfile.objects.get(user=self.user) profile.metadata = dict() profile.save() - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) - data = { - 'metadata': json_metadata, - 'overwrite': 'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": json_metadata, "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) @@ -582,505 +591,535 @@ def test_partial_updates_empty_metadata(self): def test_partial_updates_too_long(self): # the max field length for username is 30 in django - username = 'a' * 31 - data = {'username': username} - request = self.factory.patch('/', data=data, **self.extra) + username = "a" * 31 + data = {"username": username} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 400) self.assertEqual( response.data, - {'username': - [u'Ensure this field has no more than 30 characters.']}) + {"username": ["Ensure this field has no more than 30 characters."]}, + ) self.assertNotEqual(profile.user.username, username) def test_partial_update_metadata_field(self): - metadata = {u"zebra": {u"key1": "value1", u"key2": "value2"}} + metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) data = { - 'metadata': json_metadata, + "metadata": json_metadata, } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual(profile.metadata, metadata) # create a new key/value object if it doesn't exist - data = { - 'metadata': '{"zebra": {"key3": "value3"}}', - 'overwrite': u'false' - } - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key3": "value3"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "value2", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "value2", "key3": "value3"}}, + ) # update an existing key/value object - data = { - 'metadata': '{"zebra": {"key2": "second"}}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"zebra": {"key2": "second"}}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, {u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}}) + profile.metadata, + {"zebra": {"key1": "value1", "key2": "second", "key3": "value3"}}, + ) # add a new key/value object if the key doesn't exist - data = { - 'metadata': '{"animal": "donkey"}', 'overwrite': u'false'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"animal": "donkey"}', "overwrite": "false"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) self.assertEqual( - profile.metadata, { - u"zebra": { - u"key1": "value1", u"key2": "second", u"key3": "value3"}, - u'animal': u'donkey'}) + profile.metadata, + { + "zebra": {"key1": "value1", "key2": "second", "key3": "value3"}, + "animal": "donkey", + }, + ) # don't pass overwrite param - data = {'metadata': '{"b": "caah"}'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}'} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) # pass 'overwrite' param whose value isn't false - data = {'metadata': '{"b": "caah"}', 'overwrite': u'falsey'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"metadata": '{"b": "caah"}', "overwrite": "falsey"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) profile = UserProfile.objects.get(user=self.user) self.assertEqual(response.status_code, 200) - self.assertEqual( - profile.metadata, {u'b': u'caah'}) + self.assertEqual(profile.metadata, {"b": "caah"}) def test_put_update(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # edit username with existing different user's username - data['username'] = 'bob' + data["username"] = "bob" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 400) # update - data['username'] = 'roger' - data['city'] = 'Nairobi' + data["username"] = "roger" + data["city"] = "Nairobi" request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) - response = self.view(request, user='deno') + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['city'], data['city']) + self.assertEqual(response.data["city"], data["city"]) def test_profile_create_mixed_case(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get( - user__username=data['username'].lower()) - data['id'] = profile.user.pk - data['gravatar'] = str(profile.gravatar) - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['username'] = u'deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"].lower()) + data["id"] = profile.user.pk + data["gravatar"] = str(profile.gravatar) + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["username"] = "deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - data['username'] = u'deno' - data['joined_on'] = str(profile.user.date_joined) + data["username"] = "deno" + data["joined_on"] = str(profile.user.date_joined) request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertIn("%s already exists" % - data['username'], response.data['username']) + self.assertIn("%s already exists" % data["username"], response.data["username"]) def test_change_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "bobbob" new_password = "bobbob1" old_token = Token.objects.get(user=self.user).key - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") now = timezone.now().isoformat() user = User.objects.get(username__iexact=self.user.username) user_profile = UserProfile.objects.get(user_id=user.id) self.assertEqual(response.status_code, 200) self.assertEqual( - type(parse_datetime(user_profile.metadata['last_password_edit'])), - type(parse_datetime(now))) + type(parse_datetime(user_profile.metadata["last_password_edit"])), + type(parse_datetime(now)), + ) self.assertTrue(user.check_password(new_password)) - self.assertIn('access_token', response.data) - self.assertIn('temp_token', response.data) - self.assertEqual(response.data['username'], self.user.username) - self.assertNotEqual(response.data['access_token'], old_token) + self.assertIn("access_token", response.data) + self.assertIn("temp_token", response.data) + self.assertEqual(response.data["username"], self.user.username) + self.assertNotEqual(response.data["access_token"], old_token) # Assert requests made with the old tokens are rejected - post_data = { - 'current_password': new_password, - 'new_password': 'random'} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": new_password, "new_password": "random"} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 401) def test_change_password_wrong_current_password(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} + post_data = {"current_password": current_password, "new_password": new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") user = User.objects.get(username__iexact=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual( - response.data, - "Invalid password. You have 9 attempts left.") + self.assertEqual(response.data, "Invalid password. You have 9 attempts left.") self.assertFalse(user.check_password(new_password)) def test_profile_create_with_name(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - del data['password'] - profile = UserProfile.objects.get(user__username=data['username']) - data['id'] = profile.user.pk - data['first_name'] = 'Dennis' - data['last_name'] = 'deno' - data['gravatar'] = profile.gravatar - data['url'] = 'http://testserver/api/v1/profiles/deno' - data['user'] = 'http://testserver/api/v1/users/deno' - data['metadata'] = {} - data['metadata']['last_password_edit'] = \ - profile.metadata['last_password_edit'] - data['joined_on'] = profile.user.date_joined + del data["password"] + profile = UserProfile.objects.get(user__username=data["username"]) + data["id"] = profile.user.pk + data["first_name"] = "Dennis" + data["last_name"] = "deno" + data["gravatar"] = profile.gravatar + data["url"] = "http://testserver/api/v1/profiles/deno" + data["user"] = "http://testserver/api/v1/users/deno" + data["metadata"] = {} + data["metadata"]["last_password_edit"] = profile.metadata["last_password_edit"] + data["joined_on"] = profile.user.date_joined self.assertEqual(response.data, data) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_twitter_username_validation(self): data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoerama', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoerama", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - data['twitter'] = 'denoerama' + data["twitter"] = "denoerama" data = { - 'username': u'deno', - 'name': u'Dennis deno', - 'email': u'deno@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'organization': u'Dono Inc.', - 'website': u'deno.com', - 'twitter': u'denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs', - 'require_auth': False, - 'password': 'denodeno', - 'is_org': False, + "username": "deno", + "name": "Dennis deno", + "email": "deno@columbia.edu", + "city": "Denoville", + "country": "US", + "organization": "Dono Inc.", + "website": "deno.com", + "twitter": "denoeramaddfsdsl8729320392ujijdswkp--22kwklskdsjs", + "require_auth": False, + "password": "denodeno", + "is_org": False, } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['twitter'], - [u'Invalid twitter username {}'.format(data['twitter'])] + response.data["twitter"], + ["Invalid twitter username {}".format(data["twitter"])], ) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_put_patch_method_on_names(self): data = _profile_data() # create profile request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # update - data['first_name'] = 'Tom' - del data['name'] + data["first_name"] = "Tom" + del data["name"] request = self.factory.put( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) - response = self.view(request, user='deno') + response = self.view(request, user="deno") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) - first_name = u'Henry' - last_name = u'Thierry' + first_name = "Henry" + last_name = "Thierry" - data = {'first_name': first_name, 'last_name': last_name} - request = self.factory.patch('/', data=data, **self.extra) + data = {"first_name": first_name, "last_name": last_name} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['first_name'], data['first_name']) - self.assertEqual(response.data['last_name'], data['last_name']) + self.assertEqual(response.data["first_name"], data["first_name"]) + self.assertEqual(response.data["last_name"], data["last_name"]) - @patch('django.core.mail.EmailMultiAlternatives.send') + @patch("django.core.mail.EmailMultiAlternatives.send") def test_send_email_activation_api(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) # Activation email not sent self.assertFalse(mock_send_mail.called) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") self.assertTrue(user.is_active) def test_partial_update_without_password_fails(self): - data = {'email': 'user@example.com'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example.com"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) self.assertEqual( - [u'Your password is required when updating your email address.'], - response.data) + ["Your password is required when updating your email address."], + response.data, + ) def test_partial_update_with_invalid_email_fails(self): - data = {'email': 'user@example'} - request = self.factory.patch('/', data=data, **self.extra) + data = {"email": "user@example"} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 400) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) def test_partial_update_email(self, mock_send_verification_email): profile_data = _profile_data() self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'user@example.com', - 'password': "invalid_password"} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": "invalid_password"} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) self.assertEqual(response.status_code, 400) - data = {'email': 'user@example.com', - 'password': profile_data.get('password')} - request = self.factory.patch('/', data=data, **deno_extra) - response = self.view(request, user=profile_data.get('username')) + data = {"email": "user@example.com", "password": profile_data.get("password")} + request = self.factory.patch("/", data=data, **deno_extra) + response = self.view(request, user=profile_data.get("username")) profile = UserProfile.objects.get(user=rp.user) self.assertEqual(response.status_code, 200) - self.assertEqual(profile.user.email, 'user@example.com') + self.assertEqual(profile.user.email, "user@example.com") rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) def test_update_first_last_name_password_not_affected(self): - data = {'first_name': 'update_first', - 'last_name': 'update_last'} + data = {"first_name": "update_first", "last_name": "update_last"} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=self.user.username) self.assertEqual(response.status_code, 200) view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bobbob') + auth = DigestAuth("bob@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) @patch( - ('onadata.libs.serializers.user_profile_serializer.' - 'send_verification_email.delay') + ( + "onadata.libs.serializers.user_profile_serializer." + "send_verification_email.delay" + ) ) - def test_partial_update_unique_email_api( - self, mock_send_verification_email): + def test_partial_update_unique_email_api(self, mock_send_verification_email): profile_data = { - 'username': u'bobby', - 'first_name': u'Bob', - 'last_name': u'Blender', - 'email': u'bobby@columbia.edu', - 'city': u'Bobville', - 'country': u'US', - 'organization': u'Bob Inc.', - 'website': u'bob.com', - 'twitter': u'boberama', - 'require_auth': False, - 'password': 'bobbob', - 'is_org': False, - 'name': u'Bob Blender' + "username": "bobby", + "first_name": "Bob", + "last_name": "Blender", + "email": "bobby@columbia.edu", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "website": "bob.com", + "twitter": "boberama", + "require_auth": False, + "password": "bobbob", + "is_org": False, + "name": "Bob Blender", } self._create_user_using_profiles_endpoint(profile_data) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - deno_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % rp.user.auth_token - } + deno_extra = {"HTTP_AUTHORIZATION": "Token %s" % rp.user.auth_token} - data = {'email': 'example@gmail.com', - 'password': profile_data.get('password')} + data = {"email": "example@gmail.com", "password": profile_data.get("password")} request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **deno_extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **deno_extra + ) response = self.view(request, user=rp.user.username) self.assertEqual(response.status_code, 200) rp = RegistrationProfile.objects.get( - user__username=profile_data.get('username') + user__username=profile_data.get("username") ) - self.assertIn('is_email_verified', rp.user.profile.metadata) - self.assertFalse(rp.user.profile.metadata.get('is_email_verified')) + self.assertIn("is_email_verified", rp.user.profile.metadata) + self.assertFalse(rp.user.profile.metadata.get("is_email_verified")) self.assertTrue(mock_send_verification_email.called) - self.assertEqual(response.data['email'], data['email']) + self.assertEqual(response.data["email"], data["email"]) # create User request = self.factory.post( - '/api/v1/profiles', data=json.dumps(_profile_data()), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(_profile_data()), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - user = User.objects.get(username='deno') + user = User.objects.get(username="deno") # Update email request = self.factory.patch( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request, user=user.username) self.assertEqual(response.status_code, 400) def test_profile_create_fails_with_long_first_and_last_names(self): data = { - 'username': u'machicimo', - 'email': u'mike@columbia.edu', - 'city': u'Denoville', - 'country': u'US', - 'last_name': - u'undeomnisistenatuserrorsitvoluptatem', - 'first_name': - u'quirationevoluptatemsequinesciunt' + "username": "machicimo", + "email": "mike@columbia.edu", + "city": "Denoville", + "country": "US", + "last_name": "undeomnisistenatuserrorsitvoluptatem", + "first_name": "quirationevoluptatemsequinesciunt", } request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) - self.assertEqual(response.data['first_name'][0], - u'Ensure this field has no more than 30 characters.') - self.assertEqual(response.data['last_name'][0], - u'Ensure this field has no more than 30 characters.') + self.assertEqual( + response.data["first_name"][0], + "Ensure this field has no more than 30 characters.", + ) + self.assertEqual( + response.data["last_name"][0], + "Ensure this field has no more than 30 characters.", + ) self.assertEqual(response.status_code, 400) @all_requests def grant_perms_form_builder(self, url, request): - assert 'Authorization' in request.headers - assert request.headers.get('Authorization').startswith('Token') + assert "Authorization" in request.headers + assert request.headers.get("Authorization").startswith("Token") response = requests.Response() response.status_code = 201 - response._content = \ - { - "detail": "Successfully granted default model level perms to" - " user." - } + response._content = { + "detail": "Successfully granted default model level perms to" " user." + } return response def test_create_user_with_given_name(self): registered_functions = [r[1]() for r in signals.post_save.receivers] self.assertIn(set_kpi_formbuilder_permissions, registered_functions) with HTTMock(self.grant_perms_form_builder): - with self.settings(KPI_FORMBUILDER_URL='http://test_formbuilder$'): + with self.settings(KPI_FORMBUILDER_URL="http://test_formbuilder$"): extra_data = {"username": "rust"} self._login_user_and_profile(extra_post_data=extra_data) @@ -1088,110 +1127,137 @@ def test_get_monthly_submissions(self): """ Test getting monthly submissions for a user """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() count1 = Instance.objects.filter(xform=self.xform).count() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count1}) + self.assertEquals(response.data, {"private": count1}) # publish another form, make submission and make it public self._publish_form_with_hxl_support() - self.assertEquals(self.xform.id_string, 'hxl_example') - count2 = Instance.objects.filter(xform=self.xform).filter( - date_created__year=datetime.datetime.now().year).filter( - date_created__month=datetime.datetime.now().month).filter( - date_created__day=datetime.datetime.now().day).count() + self.assertEquals(self.xform.id_string, "hxl_example") + count2 = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=datetime.datetime.now().year) + .filter(date_created__month=datetime.datetime.now().month) + .filter(date_created__day=datetime.datetime.now().day) + .count() + ) self.xform.shared = True self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, {'private': count1, 'public': count2}) + self.assertEquals(response.data, {"private": count1, "public": count2}) def test_get_monthly_submissions_with_year_and_month_params(self): """ Test passing both month and year params """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) # publish form and make a submission dated 2013-02-18 self._publish_xls_form_to_project() survey = self.surveys[0] - submission_time = parse_datetime('2013-02-18 15:54:01Z') + submission_time = parse_datetime("2013-02-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=submission_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__month=2).filter(date_created__year=2013).count() + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=submission_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__month=2) + .filter(date_created__year=2013) + .count() + ) # get submission count and assert the response is correct - data = {'month': 2, 'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": 2, "year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) def test_monthly_submissions_with_month_param(self): """ Test that by passing only the value for month, the year is assumed to be the current year """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month year = datetime.datetime.now().year # publish form and make submissions self._publish_xls_form_to_project() self._make_submissions() - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=year).filter(date_created__month=month).count() + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=year) + .filter(date_created__month=month) + .count() + ) - data = {'month': month} - request = self.factory.get('/', data=data, **self.extra) + data = {"month": month} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) def test_monthly_submissions_with_year_param(self): """ Test that by passing only the value for year the month is assumed to be the current month """ - view = UserProfileViewSet.as_view({'get': 'monthly_submissions'}) + view = UserProfileViewSet.as_view({"get": "monthly_submissions"}) month = datetime.datetime.now().month # publish form and make submissions dated the year 2013 # and the current month self._publish_xls_form_to_project() survey = self.surveys[0] - _time = parse_datetime('2013-' + str(month) + '-18 15:54:01Z') + _time = parse_datetime("2013-" + str(month) + "-18 15:54:01Z") self._make_submission( - os.path.join(self.main_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml'), - forced_submission_time=_time) - count = Instance.objects.filter(xform=self.xform).filter( - date_created__year=2013).filter( - date_created__month=month).count() - - data = {'year': 2013} - request = self.factory.get('/', data=data, **self.extra) + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ), + forced_submission_time=_time, + ) + count = ( + Instance.objects.filter(xform=self.xform) + .filter(date_created__year=2013) + .filter(date_created__month=month) + .count() + ) + + data = {"year": 2013} + request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) self.assertEquals(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {'private': count}) + self.assertEquals(response.data, {"private": count}) @override_settings(ENABLE_EMAIL_VERIFICATION=True) - @patch( - 'onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile') + @patch("onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile") def test_reads_from_master(self, mock_rp_class): """ Test that on failure to retrieve a UserProfile in the first @@ -1200,88 +1266,90 @@ def test_reads_from_master(self, mock_rp_class): data = _profile_data() self._create_user_using_profiles_endpoint(data) - view = UserProfileViewSet.as_view({'get': 'verify_email'}) - rp = RegistrationProfile.objects.get( - user__username=data.get('username') - ) - _data = {'verification_key': rp.activation_key} + view = UserProfileViewSet.as_view({"get": "verify_email"}) + rp = RegistrationProfile.objects.get(user__username=data.get("username")) + _data = {"verification_key": rp.activation_key} mock_rp_class.DoesNotExist = RegistrationProfile.DoesNotExist mock_rp_class.objects.select_related( - 'user', 'user__profile' - ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] - request = self.factory.get('/', data=_data) + "user", "user__profile" + ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] + request = self.factory.get("/", data=_data) response = view(request) self.assertEquals(response.status_code, 200) - self.assertIn('is_email_verified', response.data) - self.assertIn('username', response.data) - self.assertTrue(response.data.get('is_email_verified')) - self.assertEquals( - response.data.get('username'), data.get('username')) + self.assertIn("is_email_verified", response.data) + self.assertIn("username", response.data) + self.assertTrue(response.data.get("is_email_verified")) + self.assertEquals(response.data.get("username"), data.get("username")) self.assertEqual( mock_rp_class.objects.select_related( - 'user', 'user__profile').get.call_count, 2) + "user", "user__profile" + ).get.call_count, + 2, + ) def test_change_password_attempts(self): - view = UserProfileViewSet.as_view( - {'post': 'change_password'}) + view = UserProfileViewSet.as_view({"post": "change_password"}) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') - self.assertIsNone(cache.get('change_password_attempts-bob')) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") + self.assertIsNone(cache.get("change_password_attempts-bob")) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) # first attempt current_password = "wrong_pass" new_password = "bobbob1" - post_data = {'current_password': current_password, - 'new_password': new_password} - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + post_data = {"current_password": current_password, "new_password": new_password} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password." - u" You have 9 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 1) + self.assertEqual( + response.data, "Invalid password." " You have 9 attempts left." + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 1) # second attempt - request = self.factory.post('/', data=post_data, **self.extra) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Invalid password. You have 8 attempts left.") - self.assertEqual(cache.get('change_password_attempts-bob'), 2) + self.assertEqual(response.data, "Invalid password. You have 8 attempts left.") + self.assertEqual(cache.get("change_password_attempts-bob"), 2) # check user is locked out - request = self.factory.post('/', data=post_data, **self.extra) - cache.set('change_password_attempts-bob', 9) - self.assertIsNone(cache.get('lockout_change_password_user-bob')) - response = view(request, user='bob') + request = self.factory.post("/", data=post_data, **self.extra) + cache.set("change_password_attempts-bob", 9) + self.assertIsNone(cache.get("lockout_change_password_user-bob")) + response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - "Too many password reset attempts," - u" Try again in 30 minutes") - self.assertEqual(cache.get('change_password_attempts-bob'), 10) - self.assertIsNotNone(cache.get('lockout_change_password_user-bob')) + self.assertEqual( + response.data, + "Too many password reset attempts," " Try again in 30 minutes", + ) + self.assertEqual(cache.get("change_password_attempts-bob"), 10) + self.assertIsNotNone(cache.get("lockout_change_password_user-bob")) lockout = datetime.datetime.strptime( - cache.get('lockout_change_password_user-bob'), '%Y-%m-%dT%H:%M:%S') + cache.get("lockout_change_password_user-bob"), "%Y-%m-%dT%H:%M:%S" + ) self.assertIsInstance(lockout, datetime.datetime) # clear cache - cache.delete('change_password_attempts-bob') - cache.delete('lockout_change_password_user-bob') + cache.delete("change_password_attempts-bob") + cache.delete("lockout_change_password_user-bob") - @patch('onadata.apps.main.signals.send_generic_email') + @patch("onadata.apps.main.signals.send_generic_email") @override_settings(ENABLE_ACCOUNT_ACTIVATION_EMAILS=True) def test_account_activation_emails(self, mock_send_mail): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) data = _profile_data() - del data['name'] + del data["name"] request = self.factory.post( - '/api/v1/profiles', data=json.dumps(data), - content_type="application/json", **self.extra) + "/api/v1/profiles", + data=json.dumps(data), + content_type="application/json", + **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -1292,7 +1360,7 @@ def test_account_activation_emails(self, mock_send_mail): mock_send_mail.assert_has_calls( [ call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been " "successfully created! Kindly wait till an " "administrator activates your account." @@ -1300,14 +1368,14 @@ def test_account_activation_emails(self, mock_send_mail): "Please contact us with any questions, " "we're always happy to help." "\n\nThanks,\nThe Team at Ona", - "Ona account created - Pending activation" + "Ona account created - Pending activation", ), call( - data['email'], + data["email"], "\nHi deno,\n\nYour account has been activated." "\n\nThank you for choosing Ona!" "\n\nThanks,\nThe Team at Ona", - "Ona account activated" - ) + "Ona account activated", + ), ] ) diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index d6ed815c01..3fd81192de 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -21,7 +21,7 @@ from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ from django.utils.module_loading import import_string -from future.utils import listitems +from six import iteritems from guardian.shortcuts import assign_perm, get_perms_for_model, remove_perm from guardian.shortcuts import get_perms from kombu.exceptions import OperationalError @@ -31,7 +31,9 @@ from multidb.pinning import use_master from onadata.apps.api.models.organization_profile import ( - OrganizationProfile, create_owner_team_and_assign_permissions) + OrganizationProfile, + create_owner_team_and_assign_permissions, +) from onadata.apps.api.models.team import Team from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.forms import QuickConverter @@ -41,20 +43,41 @@ from onadata.libs.baseviewset import DefaultBaseViewset from onadata.libs.models.share_project import ShareProject from onadata.libs.permissions import ( - ROLES, DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole, - EditorMinorRole, EditorRole, ManagerRole, OwnerRole, get_role, - get_role_in_org, is_organization) + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + get_role, + get_role_in_org, + is_organization, +) from onadata.libs.utils.api_export_tools import custom_response_handler from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_OWNER_CACHE, PROJ_SUB_DATE_CACHE, reset_project_cache, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_SUB_DATE_CACHE, + reset_project_cache, + safe_delete, +) from onadata.libs.utils.common_tags import MEMBERS, XFORM_META_PERMS -from onadata.libs.utils.logger_tools import (publish_form, - response_with_mimetype_and_name) -from onadata.libs.utils.project_utils import (set_project_perms_to_xform, - set_project_perms_to_xform_async) -from onadata.libs.utils.user_auth import (check_and_set_form_by_id, - check_and_set_form_by_id_string) +from onadata.libs.utils.logger_tools import ( + publish_form, + response_with_mimetype_and_name, +) +from onadata.libs.utils.project_utils import ( + set_project_perms_to_xform, + set_project_perms_to_xform_async, +) +from onadata.libs.utils.user_auth import ( + check_and_set_form_by_id, + check_and_set_form_by_id_string, +) DECIMAL_PRECISION = 2 @@ -62,18 +85,21 @@ def _get_first_last_names(name): name_split = name.split() first_name = name_split[0] - last_name = u'' + last_name = "" if len(name_split) > 1: - last_name = u' '.join(name_split[1:]) + last_name = " ".join(name_split[1:]) return first_name, last_name def _get_id_for_type(record, mongo_field): date_field = datetime_from_str(record[mongo_field]) - mongo_str = '$' + mongo_field + mongo_str = "$" + mongo_field - return {"$substr": [mongo_str, 0, 10]} if isinstance(date_field, datetime)\ + return ( + {"$substr": [mongo_str, 0, 10]} + if isinstance(date_field, datetime) else mongo_str + ) def get_accessible_forms(owner=None, shared_form=False, shared_data=False): @@ -88,13 +114,14 @@ def get_accessible_forms(owner=None, shared_form=False, shared_data=False): if shared_form and not shared_data: xforms = xforms.filter(shared=True) - elif (shared_form and shared_data) or \ - (owner == 'public' and not shared_form and not shared_data): + elif (shared_form and shared_data) or ( + owner == "public" and not shared_form and not shared_data + ): xforms = xforms.filter(Q(shared=True) | Q(shared_data=True)) elif not shared_form and shared_data: xforms = xforms.filter(shared_data=True) - if owner != 'public': + if owner != "public": xforms = xforms.filter(user__username=owner) return xforms.distinct() @@ -109,33 +136,29 @@ def create_organization(name, creator): """ organization, _created = User.objects.get_or_create(username__iexact=name) organization_profile = OrganizationProfile.objects.create( - user=organization, creator=creator) + user=organization, creator=creator + ) return organization_profile def create_organization_object(org_name, creator, attrs=None): - '''Creates an OrganizationProfile object without saving to the database''' + """Creates an OrganizationProfile object without saving to the database""" attrs = attrs if attrs else {} - name = attrs.get('name', org_name) if attrs else org_name + name = attrs.get("name", org_name) if attrs else org_name first_name, last_name = _get_first_last_names(name) - email = attrs.get('email', u'') if attrs else u'' + email = attrs.get("email", "") if attrs else "" new_user = User( username=org_name, first_name=first_name, last_name=last_name, email=email, - is_active=getattr( - settings, - 'ORG_ON_CREATE_IS_ACTIVE', - True)) + is_active=getattr(settings, "ORG_ON_CREATE_IS_ACTIVE", True), + ) new_user.save() try: - registration_profile = RegistrationProfile.objects.create_profile( - new_user) + registration_profile = RegistrationProfile.objects.create_profile(new_user) except IntegrityError: - raise ValidationError(_( - u"%s already exists" % org_name - )) + raise ValidationError(_("%s already exists" % org_name)) if email: site = Site.objects.get(pk=settings.SITE_ID) registration_profile.send_activation_email(site) @@ -144,11 +167,12 @@ def create_organization_object(org_name, creator, attrs=None): name=name, creator=creator, created_by=creator, - city=attrs.get('city', u''), - country=attrs.get('country', u''), - organization=attrs.get('organization', u''), - home_page=attrs.get('home_page', u''), - twitter=attrs.get('twitter', u'')) + city=attrs.get("city", ""), + country=attrs.get("country", ""), + organization=attrs.get("organization", ""), + home_page=attrs.get("home_page", ""), + twitter=attrs.get("twitter", ""), + ) return profile @@ -157,15 +181,18 @@ def create_organization_team(organization, name, permission_names=None): Creates an organization team with the given permissions as defined in permission_names. """ - organization = organization.user \ - if isinstance(organization, OrganizationProfile) else organization + organization = ( + organization.user + if isinstance(organization, OrganizationProfile) + else organization + ) team = Team.objects.create(organization=organization, name=name) - content_type = ContentType.objects.get( - app_label='api', model='organizationprofile') + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") if permission_names: # get permission objects perms = Permission.objects.filter( - codename__in=permission_names, content_type=content_type) + codename__in=permission_names, content_type=content_type + ) if perms: team.permissions.add(*tuple(perms)) return team @@ -176,8 +203,7 @@ def get_organization_members_team(organization): create members team if it does not exist and add organization owner to the members team""" try: - team = Team.objects.get(name=u'%s#%s' % (organization.user.username, - MEMBERS)) + team = Team.objects.get(name="%s#%s" % (organization.user.username, MEMBERS)) except Team.DoesNotExist: team = create_organization_team(organization, MEMBERS) add_user_to_team(team, organization.user) @@ -192,14 +218,14 @@ def get_or_create_organization_owners_team(org): :param org: organization :return: Owners team of the organization """ - team_name = f'{org.user.username}#{Team.OWNER_TEAM_NAME}' + team_name = f"{org.user.username}#{Team.OWNER_TEAM_NAME}" try: team = Team.objects.get(name=team_name, organization=org.user) except Team.DoesNotExist: from multidb.pinning import use_master # pylint: disable=import-error + with use_master: - queryset = Team.objects.filter( - name=team_name, organization=org.user) + queryset = Team.objects.filter(name=team_name, organization=org.user) if queryset.count() > 0: return queryset.first() # pylint: disable=no-member return create_owner_team_and_assign_permissions(org) @@ -234,12 +260,11 @@ def remove_user_from_team(team, user): user.groups.remove(team) # remove the permission - remove_perm('view_team', user, team) + remove_perm("view_team", user, team) # if team is owners team remove more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: - owners_team = get_or_create_organization_owners_team( - team.organization.profile) + owners_team = get_or_create_organization_owners_team(team.organization.profile) members_team = get_organization_members_team(team.organization.profile) for perm in get_perms_for_model(Team): remove_perm(perm.codename, user, owners_team) @@ -260,7 +285,7 @@ def add_user_to_team(team, user): user.groups.add(team) # give the user perms to view the team - assign_perm('view_team', user, team) + assign_perm("view_team", user, team) # if team is owners team assign more perms if team.name.find(Team.OWNER_TEAM_NAME) > 0: @@ -293,10 +318,8 @@ def _get_owners(organization): return [ user - for user in get_or_create_organization_owners_team( - organization).user_set.all() - if get_role_in_org(user, organization) == 'owner' - and organization.user != user + for user in get_or_create_organization_owners_team(organization).user_set.all() + if get_role_in_org(user, organization) == "owner" and organization.user != user ] @@ -318,7 +341,8 @@ def create_organization_project(organization, project_name, created_by): name=project_name, organization=organization, created_by=created_by, - metadata='{}') + metadata="{}", + ) return project @@ -343,7 +367,8 @@ def publish_xlsform(request, owner, id_string=None, project=None): Publishes XLSForm & creates an XFormVersion object given a request. """ survey = do_publish_xlsform( - request.user, request.data, request.FILES, owner, id_string, project) + request.user, request.data, request.FILES, owner, id_string, project + ) return survey @@ -354,18 +379,20 @@ def do_publish_xlsform(user, post, files, owner, id_string=None, project=None): """ if id_string and project: xform = get_object_or_404( - XForm, user=owner, id_string=id_string, project=project) + XForm, user=owner, id_string=id_string, project=project + ) if not ManagerRole.user_has_role(user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - user, xform))) - elif not user.has_perm('can_add_xform', owner.profile): + _("{} has no manager/owner role to the form {}".format(user, xform)) + ) + elif not user.has_perm("can_add_xform", owner.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % { - 'user': user.username, - 'account': owner.username - })) + detail=_( + "User %(user)s has no permission to add xforms to " + "account %(account)s" + % {"user": user.username, "account": owner.username} + ) + ) def set_form(): """ @@ -373,8 +400,8 @@ def set_form(): """ if project: - args = (post and dict(listitems(post))) or {} - args['project'] = project.pk + args = (post and dict(list(iteritems(post)))) or {} + args["project"] = project.pk else: args = post @@ -395,11 +422,11 @@ def set_form(): Instantiates QuickConverter form to publish a form. """ props = { - 'project': project.pk, - 'dropbox_xls_url': request.data.get('dropbox_xls_url'), - 'xls_url': request.data.get('xls_url'), - 'csv_url': request.data.get('csv_url'), - 'text_xls_form': request.data.get('text_xls_form') + "project": project.pk, + "dropbox_xls_url": request.data.get("dropbox_xls_url"), + "xls_url": request.data.get("xls_url"), + "csv_url": request.data.get("csv_url"), + "text_xls_form": request.data.get("text_xls_form"), } form = QuickConverter(props, request.FILES) @@ -414,30 +441,32 @@ def id_string_exists_in_account(): otherwise returns False. """ try: - XForm.objects.get( - user=project.organization, id_string=xform.id_string) + XForm.objects.get(user=project.organization, id_string=xform.id_string) except XForm.DoesNotExist: return False return True - if 'formid' in request.data: - xform = get_object_or_404(XForm, pk=request.data.get('formid')) - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, xform.project.pk)) + if "formid" in request.data: + xform = get_object_or_404(XForm, pk=request.data.get("formid")) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_FORMS_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) + safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, xform.project.pk)) if not ManagerRole.user_has_role(request.user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format( - request.user, xform))) - - msg = 'Form with the same id_string already exists in this account' + _( + "{} has no manager/owner role to the form {}".format( + request.user, xform + ) + ) + ) + + msg = "Form with the same id_string already exists in this account" # Without this check, a user can't transfer a form to projects that # he/she owns because `id_string_exists_in_account` will always # return true - if project.organization != xform.user and \ - id_string_exists_in_account(): + if project.organization != xform.user and id_string_exists_in_account(): raise exceptions.ParseError(_(msg)) xform.user = project.organization xform.project = project @@ -483,7 +512,8 @@ def get_xform(formid, request, username=None): if not xform: raise exceptions.PermissionDenied( - _("You do not have permission to view data from this form.")) + _("You do not have permission to view data from this form.") + ) return xform @@ -529,12 +559,13 @@ class TagForm(forms.Form): """ Simple TagForm class to validate tags in a request. """ + tags = TagField() form = TagForm(request.data) if form.is_valid(): - tags = form.cleaned_data.get('tags', None) + tags = form.cleaned_data.get("tags", None) if tags: for tag in tags: @@ -559,9 +590,9 @@ def get_data_value_objects(value): Looks for 'dataview 123 fruits.csv' or 'xform 345 fruits.csv'. """ model = None - if value.startswith('dataview'): + if value.startswith("dataview"): model = DataView - elif value.startswith('xform'): + elif value.startswith("xform"): model = XForm if model: @@ -575,8 +606,8 @@ def get_data_value_objects(value): if metadata.data_file: file_path = metadata.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") dfs = get_storage_class()() if dfs.exists(file_path): @@ -586,7 +617,8 @@ def get_data_value_objects(value): extension=extension, show_date=False, file_path=file_path, - full_mime=True) + full_mime=True, + ) return response return HttpResponseNotFound() @@ -600,10 +632,12 @@ def get_data_value_objects(value): return custom_response_handler( request, - xform, {}, + xform, + {}, Export.CSV_EXPORT, filename=filename, - dataview=dataview) + dataview=dataview, + ) return HttpResponseRedirect(metadata.data_value) @@ -615,7 +649,7 @@ def check_inherit_permission_from_project(xform_id, user): if there is a difference applies the project permissions to the user for the given xform_id. """ - if xform_id == 'public': + if xform_id == "public": return try: @@ -624,8 +658,12 @@ def check_inherit_permission_from_project(xform_id, user): return # get the project_xform - xform = XForm.objects.filter(pk=xform_id).select_related('project').only( - 'project_id', 'id').first() + xform = ( + XForm.objects.filter(pk=xform_id) + .select_related("project") + .only("project_id", "id") + .first() + ) if not xform: return @@ -656,8 +694,11 @@ def get_baseviewset_class(): the default in onadata :return: the default baseviewset """ - return import_string(settings.BASE_VIEWSET) \ - if settings.BASE_VIEWSET else DefaultBaseViewset + return ( + import_string(settings.BASE_VIEWSET) + if settings.BASE_VIEWSET + else DefaultBaseViewset + ) def generate_tmp_path(uploaded_csv_file): @@ -694,31 +735,31 @@ def get_xform_users(xform): org_members = get_team_members(user.username) data[user] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user in data: - data[perm.user]['permissions'].append(perm.permission.codename) + data[perm.user]["permissions"].append(perm.permission.codename) for user in org_members: if user not in data: data[user] = { - 'permissions': get_perms(user, xform), - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": get_perms(user, xform), + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } for k in data: - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], xform) - del data[k]['permissions'] + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], xform) + del data[k]["permissions"] return data @@ -731,8 +772,7 @@ def get_team_members(org_username): """ members = [] try: - team = Team.objects.get( - name="{}#{}".format(org_username, MEMBERS)) + team = Team.objects.get(name="{}#{}".format(org_username, MEMBERS)) except Team.DoesNotExist: pass else: @@ -750,20 +790,18 @@ def update_role_by_meta_xform_perms(xform): editor_role_list = [EditorRole, EditorMinorRole] editor_role = {role.name: role for role in editor_role_list} - dataentry_role_list = [ - DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole - ] + dataentry_role_list = [DataEntryMinorRole, DataEntryOnlyRole, DataEntryRole] dataentry_role = {role.name: role for role in dataentry_role_list} if metadata: - meta_perms = metadata.data_value.split('|') + meta_perms = metadata.data_value.split("|") # update roles users = get_xform_users(xform) for user in users: - role = users.get(user).get('role') + role = users.get(user).get("role") if role in editor_role: role = ROLES.get(meta_perms[0]) role.add(user, xform) @@ -777,12 +815,13 @@ def replace_attachment_name_with_url(data): site_url = Site.objects.get_current().domain for record in data: - attachments: dict = record.json.get('_attachments') + attachments: dict = record.json.get("_attachments") if attachments: attachment_details = [ - (attachment['name'], attachment['download_url']) + (attachment["name"], attachment["download_url"]) for attachment in attachments - if 'download_url' in attachment] + if "download_url" in attachment + ] question_keys = list(record.json.keys()) question_values = list(record.json.values()) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 16db8e1c42..71f8b9a364 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -5,7 +5,7 @@ import datetime import json -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.conf import settings diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index ea4efecfa9..3efe759183 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -23,7 +23,7 @@ from django.views.decorators.cache import never_cache from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse try: from multidb.pinning import use_master diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 0721fb46dd..af7e9c1c70 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -11,37 +11,47 @@ from django.db import connection from django.db.models.signals import post_delete, post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ +from six import python_2_unicode_compatible from onadata.apps.viewer.parsed_instance_tools import get_where_clause -from onadata.libs.models.sorting import (json_order_by, json_order_by_params, - sort_from_mongo_sort_str) -from onadata.libs.utils.cache_tools import (DATAVIEW_COUNT, - DATAVIEW_LAST_SUBMISSION_TIME, - XFORM_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, - safe_delete) -from onadata.libs.utils.common_tags import (ATTACHMENTS, EDITED, GEOLOCATION, - ID, LAST_EDITED, MONGO_STRFTIME, - NOTES, SUBMISSION_TIME) - -SUPPORTED_FILTERS = ['=', '>', '<', '>=', '<=', '<>', '!='] -ATTACHMENT_TYPES = ['photo', 'audio', 'video'] -DEFAULT_COLUMNS = [ - ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] - - -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): - _json_str = u"json->>%s" +from onadata.libs.models.sorting import ( + json_order_by, + json_order_by_params, + sort_from_mongo_sort_str, +) +from onadata.libs.utils.cache_tools import ( + DATAVIEW_COUNT, + DATAVIEW_LAST_SUBMISSION_TIME, + XFORM_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + ATTACHMENTS, + EDITED, + GEOLOCATION, + ID, + LAST_EDITED, + MONGO_STRFTIME, + NOTES, + SUBMISSION_TIME, +) + +SUPPORTED_FILTERS = ["=", ">", "<", ">=", "<=", "<>", "!="] +ATTACHMENT_TYPES = ["photo", "audio", "video"] +DEFAULT_COLUMNS = [ID, SUBMISSION_TIME, EDITED, LAST_EDITED, NOTES] + + +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): + _json_str = "json->>%s" if known_integers is not None and key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif known_dates is not None and key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif known_decimals is not None and key in known_decimals: - _json_str = u"CAST(JSON->>%s AS DECIMAL)" + _json_str = "CAST(JSON->>%s AS DECIMAL)" return _json_str @@ -51,10 +61,10 @@ def get_name_from_survey_element(element): def append_where_list(comp, t_list, json_str): - if comp in ['=', '>', '<', '>=', '<=']: - t_list.append(u"{} {} %s".format(json_str, comp)) - elif comp in ['<>', '!=']: - t_list.append(u"{} <> %s".format(json_str)) + if comp in ["=", ">", "<", ">=", "<="]: + t_list.append("{} {} %s".format(json_str, comp)) + elif comp in ["<>", "!="]: + t_list.append("{} <> %s".format(json_str)) return t_list @@ -63,8 +73,7 @@ def get_elements_of_type(xform, field_type): """ This function returns a list of column names of a specified type """ - return [f.get('name') - for f in xform.get_survey_elements_of_type(field_type)] + return [f.get("name") for f in xform.get_survey_elements_of_type(field_type)] def has_attachments_fields(data_view): @@ -95,8 +104,8 @@ class DataView(models.Model): """ name = models.CharField(max_length=255) - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) - project = models.ForeignKey('logger.Project', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) + project = models.ForeignKey("logger.Project", on_delete=models.CASCADE) columns = JSONField() query = JSONField(default=dict, blank=True) @@ -105,15 +114,19 @@ class DataView(models.Model): date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='dataview_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + related_name="dataview_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) class Meta: - app_label = 'logger' - verbose_name = _('Data View') - verbose_name_plural = _('Data Views') + app_label = "logger" + verbose_name = _("Data View") + verbose_name_plural = _("Data Views") def __str__(self): return getattr(self, "name", "") @@ -146,35 +159,39 @@ def _get_known_type(self, type_str): # pylint: disable=E1101 return [ get_name_from_survey_element(e) - for e in self.xform.get_survey_elements_of_type(type_str)] + for e in self.xform.get_survey_elements_of_type(type_str) + ] def get_known_integers(self): """Return elements of type integer""" - return self._get_known_type('integer') + return self._get_known_type("integer") def get_known_dates(self): """Return elements of type date""" - return self._get_known_type('date') + return self._get_known_type("date") def get_known_decimals(self): """Return elements of type decimal""" - return self._get_known_type('decimal') + return self._get_known_type("decimal") def has_instance(self, instance): """Return True if instance in set of dataview data""" cursor = connection.cursor() - sql = u"SELECT count(json) FROM logger_instance" - - where, where_params = self._get_where_clause(self, - self.get_known_integers(), - self.get_known_dates(), - self.get_known_decimals()) - sql_where = u"" + sql = "SELECT count(json) FROM logger_instance" + + where, where_params = self._get_where_clause( + self, + self.get_known_integers(), + self.get_known_dates(), + self.get_known_decimals(), + ) + sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE xform_id = %s AND id = %s" + sql_where \ - + u" AND deleted_at IS NULL" + sql += ( + " WHERE xform_id = %s AND id = %s" + sql_where + " AND deleted_at IS NULL" + ) # pylint: disable=E1101 params = [self.xform.pk, instance.id] + where_params @@ -192,20 +209,25 @@ def soft_delete(self, user=None): uniqueness constraint. """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix - update_fields = ['date_modified', 'deleted_at', 'name', 'deleted_by'] + update_fields = ["date_modified", "deleted_at", "name", "deleted_by"] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) @classmethod - def _get_where_clause(cls, data_view, form_integer_fields=[], - form_date_fields=[], form_decimal_fields=[]): - known_integers = ['_id'] + form_integer_fields - known_dates = ['_submission_time'] + form_date_fields + def _get_where_clause( + cls, + data_view, + form_integer_fields=[], + form_date_fields=[], + form_decimal_fields=[], + ): + known_integers = ["_id"] + form_integer_fields + known_dates = ["_submission_time"] + form_date_fields known_decimals = form_decimal_fields where = [] where_params = [] @@ -216,19 +238,19 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], or_params = [] for qu in query: - comp = qu.get('filter') - column = qu.get('column') - value = qu.get('value') - condi = qu.get('condition') + comp = qu.get("filter") + column = qu.get("column") + value = qu.get("value") + condi = qu.get("condition") - json_str = _json_sql_str(column, known_integers, known_dates, - known_decimals) + json_str = _json_sql_str( + column, known_integers, known_dates, known_decimals + ) if comp in known_dates: - value = datetime.datetime.strptime( - value[:19], MONGO_STRFTIME) + value = datetime.datetime.strptime(value[:19], MONGO_STRFTIME) - if condi and condi.lower() == 'or': + if condi and condi.lower() == "or": or_where = append_where_list(comp, or_where, json_str) or_params.extend((column, text(value))) else: @@ -236,7 +258,7 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], where_params.extend((column, text(value))) if or_where: - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] where += or_where where_params.extend(or_params) @@ -246,19 +268,18 @@ def _get_where_clause(cls, data_view, form_integer_fields=[], @classmethod def query_iterator(cls, sql, fields=None, params=[], count=False): cursor = connection.cursor() - sql_params = tuple( - i if isinstance(i, tuple) else text(i) for i in params) + sql_params = tuple(i if isinstance(i, tuple) else text(i) for i in params) if count: - from_pos = sql.upper().find(' FROM') + from_pos = sql.upper().find(" FROM") if from_pos != -1: - sql = u"SELECT COUNT(*) " + sql[from_pos:] + sql = "SELECT COUNT(*) " + sql[from_pos:] - order_pos = sql.upper().find('ORDER BY') + order_pos = sql.upper().find("ORDER BY") if order_pos != -1: sql = sql[:order_pos] - fields = [u'count'] + fields = ["count"] cursor.execute(sql, sql_params) @@ -274,16 +295,22 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): yield dict(zip(fields, [row[0].get(f) for f in fields])) @classmethod - def generate_query_string(cls, data_view, start_index, limit, - last_submission_time, all_data, sort, - filter_query=None): - additional_columns = [GEOLOCATION] \ - if data_view.instances_with_geopoints else [] + def generate_query_string( + cls, + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query=None, + ): + additional_columns = [GEOLOCATION] if data_view.instances_with_geopoints else [] if has_attachments_fields(data_view): additional_columns += [ATTACHMENTS] - sql = u"SELECT json FROM logger_instance" + sql = "SELECT json FROM logger_instance" if all_data or data_view.matches_parent: columns = None elif last_submission_time: @@ -300,12 +327,15 @@ def generate_query_string(cls, data_view, start_index, limit, data_view, data_view.get_known_integers(), data_view.get_known_dates(), - data_view.get_known_decimals()) + data_view.get_known_decimals(), + ) if filter_query: - add_where, add_where_params = \ - get_where_clause(filter_query, data_view.get_known_integers(), - data_view.get_known_decimals()) + add_where, add_where_params = get_where_clause( + filter_query, + data_view.get_known_integers(), + data_view.get_known_decimals(), + ) if add_where: where = where + add_where @@ -313,55 +343,74 @@ def generate_query_string(cls, data_view, start_index, limit, sql_where = "" if where: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) if data_view.xform.is_merged_dataset: - sql += u" WHERE xform_id IN %s " + sql_where \ - + u" AND deleted_at IS NULL" - params = [tuple(list( - data_view.xform.mergedxform.xforms.values_list('pk', flat=True) - ))] + where_params + sql += " WHERE xform_id IN %s " + sql_where + " AND deleted_at IS NULL" + params = [ + tuple( + list( + data_view.xform.mergedxform.xforms.values_list("pk", flat=True) + ) + ) + ] + where_params else: - sql += u" WHERE xform_id = %s " + sql_where \ - + u" AND deleted_at IS NULL" + sql += " WHERE xform_id = %s " + sql_where + " AND deleted_at IS NULL" params = [data_view.xform.pk] + where_params if sort is not None: - sort = ['id'] if sort is None\ - else sort_from_mongo_sort_str(sort) - sql = u"{} {}".format(sql, json_order_by(sort)) + sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) + sql = "{} {}".format(sql, json_order_by(sort)) params = params + json_order_by_params(sort) elif last_submission_time is False: - sql += ' ORDER BY id' + sql += " ORDER BY id" if start_index is not None: - sql += u" OFFSET %s" + sql += " OFFSET %s" params += [start_index] if limit is not None: - sql += u" LIMIT %s" + sql += " LIMIT %s" params += [limit] if last_submission_time: - sql += u" ORDER BY date_created DESC" - sql += u" LIMIT 1" + sql += " ORDER BY date_created DESC" + sql += " LIMIT 1" - return (sql, columns, params, ) + return ( + sql, + columns, + params, + ) @classmethod - def query_data(cls, data_view, start_index=None, limit=None, count=None, - last_submission_time=False, all_data=False, sort=None, - filter_query=None): + def query_data( + cls, + data_view, + start_index=None, + limit=None, + count=None, + last_submission_time=False, + all_data=False, + sort=None, + filter_query=None, + ): (sql, columns, params) = cls.generate_query_string( - data_view, start_index, limit, last_submission_time, - all_data, sort, filter_query) + data_view, + start_index, + limit, + last_submission_time, + all_data, + sort, + filter_query, + ) try: - records = [record for record in DataView.query_iterator(sql, - columns, - params, - count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, count) + ] except Exception as e: return {"error": _(text(e))} @@ -369,23 +418,18 @@ def query_data(cls, data_view, start_index=None, limit=None, count=None, def clear_cache(sender, instance, **kwargs): - """ Post delete handler for clearing the dataview cache. - """ - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + """Post delete handler for clearing the dataview cache.""" + safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) def clear_dataview_cache(sender, instance, **kwargs): - """ Post Save handler for clearing dataview cache on serialized fields. - """ - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, instance.project.pk)) - safe_delete('{}{}'.format(DATAVIEW_COUNT, instance.xform.pk)) - safe_delete( - '{}{}'.format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) - safe_delete('{}{}'.format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + """Post Save handler for clearing dataview cache on serialized fields.""" + safe_delete("{}{}".format(PROJ_OWNER_CACHE, instance.project.pk)) + safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform.pk)) + safe_delete("{}{}".format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) + safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) -post_save.connect(clear_dataview_cache, sender=DataView, - dispatch_uid='clear_cache') +post_save.connect(clear_dataview_cache, sender=DataView, dispatch_uid="clear_cache") -post_delete.connect(clear_cache, sender=DataView, - dispatch_uid='clear_xform_cache') +post_delete.connect(clear_cache, sender=DataView, dispatch_uid="clear_xform_cache") diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index fdd90df0b9..721fa7ff29 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -19,7 +19,7 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import ugettext as _ -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview diff --git a/onadata/apps/logger/models/open_data.py b/onadata/apps/logger/models/open_data.py index 742393c3a2..c7c85f852d 100644 --- a/onadata/apps/logger/models/open_data.py +++ b/onadata/apps/logger/models/open_data.py @@ -7,7 +7,7 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from onadata.libs.utils.common_tools import get_uuid @@ -18,11 +18,12 @@ class OpenData(models.Model): OpenData model represents a way to access private datasets without authentication using the unique uuid. """ + name = models.CharField(max_length=255) uuid = models.CharField(max_length=32, default=get_uuid, unique=True) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(null=True, blank=True) - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") active = models.BooleanField(default=True) date_created = models.DateTimeField(auto_now_add=True) @@ -32,7 +33,7 @@ def __str__(self): return getattr(self, "name", "") class Meta: - app_label = 'logger' + app_label = "logger" def get_or_create_opendata(xform): @@ -47,8 +48,8 @@ def get_or_create_opendata(xform): return OpenData.objects.get_or_create( object_id=xform.id, defaults={ - 'name': xform.id_string, - 'content_type': content_type, - 'content_object': xform, - } + "name": xform.id_string, + "content_type": content_type, + "content_object": xform, + }, ) diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index 6865da516d..81c03b1bd1 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -10,7 +10,7 @@ from django.db.models import Prefetch from django.db.models.signals import post_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -24,38 +24,59 @@ class PrefetchManager(models.Manager): def get_queryset(self): from onadata.apps.logger.models.xform import XForm from onadata.apps.api.models.team import Team - return super(PrefetchManager, self).get_queryset().select_related( - 'created_by', 'organization' - ).prefetch_related( - Prefetch('xform_set', - queryset=XForm.objects.filter(deleted_at__isnull=True) - .select_related('user') - .prefetch_related('user') - .prefetch_related('dataview_set') - .prefetch_related('metadata_set') - .only('id', 'user', 'project', 'title', 'date_created', - 'last_submission_time', 'num_of_submissions', - 'downloadable', 'id_string', 'is_merged_dataset'), - to_attr='xforms_prefetch') - ).prefetch_related('tags')\ - .prefetch_related(Prefetch( - 'projectuserobjectpermission_set', - queryset=ProjectUserObjectPermission.objects.select_related( - 'user__profile__organizationprofile', - 'permission' + + return ( + super(PrefetchManager, self) + .get_queryset() + .select_related("created_by", "organization") + .prefetch_related( + Prefetch( + "xform_set", + queryset=XForm.objects.filter(deleted_at__isnull=True) + .select_related("user") + .prefetch_related("user") + .prefetch_related("dataview_set") + .prefetch_related("metadata_set") + .only( + "id", + "user", + "project", + "title", + "date_created", + "last_submission_time", + "num_of_submissions", + "downloadable", + "id_string", + "is_merged_dataset", + ), + to_attr="xforms_prefetch", + ) + ) + .prefetch_related("tags") + .prefetch_related( + Prefetch( + "projectuserobjectpermission_set", + queryset=ProjectUserObjectPermission.objects.select_related( + "user__profile__organizationprofile", "permission" + ), + ) + ) + .prefetch_related( + Prefetch( + "projectgroupobjectpermission_set", + queryset=ProjectGroupObjectPermission.objects.select_related( + "group", "permission" + ), ) - ))\ - .prefetch_related(Prefetch( - 'projectgroupobjectpermission_set', - queryset=ProjectGroupObjectPermission.objects.select_related( - 'group', - 'permission' + ) + .prefetch_related("user_stars") + .prefetch_related( + Prefetch( + "organization__team_set", + queryset=Team.objects.all().prefetch_related("user_set"), ) - )).prefetch_related('user_stars')\ - .prefetch_related(Prefetch( - 'organization__team_set', - queryset=Team.objects.all().prefetch_related('user_set') - )) + ) + ) @python_2_unicode_compatible @@ -67,48 +88,56 @@ class Project(BaseModel): name = models.CharField(max_length=255) metadata = JSONField(default=dict) organization = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_org', - on_delete=models.CASCADE) + settings.AUTH_USER_MODEL, related_name="project_org", on_delete=models.CASCADE + ) created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, related_name='project_owner', - on_delete=models.CASCADE) - user_stars = models.ManyToManyField(settings.AUTH_USER_MODEL, - related_name='project_stars') + settings.AUTH_USER_MODEL, related_name="project_owner", on_delete=models.CASCADE + ) + user_stars = models.ManyToManyField( + settings.AUTH_USER_MODEL, related_name="project_stars" + ) shared = models.BooleanField(default=False) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) - deleted_by = models.ForeignKey(User, related_name='project_deleted_by', - blank=True, null=True, default=None, - on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, + related_name="project_deleted_by", + blank=True, + null=True, + default=None, + on_delete=models.SET_NULL, + ) objects = models.Manager() - tags = TaggableManager(related_name='project_tags') + tags = TaggableManager(related_name="project_tags") prefetched = PrefetchManager() class Meta: - app_label = 'logger' - unique_together = (('name', 'organization'),) + app_label = "logger" + unique_together = (("name", "organization"),) permissions = ( - ('add_project_xform', "Can add xform to project"), + ("add_project_xform", "Can add xform to project"), ("report_project_xform", "Can make submissions to the project"), - ('transfer_project', "Can transfer project to different owner"), - ('can_export_project_data', "Can export data in project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), ("view_project_all", "Can view all associated data"), ("view_project_data", "Can view submitted data"), ) def __str__(self): - return u'%s|%s' % (self.organization, self.name) + return "%s|%s" % (self.organization, self.name) def clean(self): # pylint: disable=E1101 - query_set = Project.objects.exclude(pk=self.pk)\ - .filter(name__iexact=self.name, organization=self.organization) + query_set = Project.objects.exclude(pk=self.pk).filter( + name__iexact=self.name, organization=self.organization + ) if query_set.exists(): - raise ValidationError(u'Project name "%s" is already in' - u' use in this account.' - % self.name.lower()) + raise ValidationError( + 'Project name "%s" is already in' + " use in this account." % self.name.lower() + ) @property def user(self): @@ -124,7 +153,7 @@ def soft_delete(self, user=None): """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.name += deletion_suffix if user is not None: @@ -140,9 +169,10 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): for perm in get_perms_for_model(Project): assign_perm(perm.codename, instance.organization, instance) - owners = instance.organization.team_set\ - .filter(name="{}#{}".format(instance.organization.username, - OWNER_TEAM_NAME), organization=instance.organization) + owners = instance.organization.team_set.filter( + name="{}#{}".format(instance.organization.username, OWNER_TEAM_NAME), + organization=instance.organization, + ) for owner in owners: assign_perm(perm.codename, owner, instance) @@ -153,8 +183,11 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -post_save.connect(set_object_permissions, sender=Project, - dispatch_uid='set_project_object_permissions') +post_save.connect( + set_object_permissions, + sender=Project, + dispatch_uid="set_project_object_permissions", +) class ProjectUserObjectPermission(UserObjectPermissionBase): diff --git a/onadata/apps/logger/models/survey_type.py b/onadata/apps/logger/models/survey_type.py index 6201c85b7d..d766b35b96 100644 --- a/onadata/apps/logger/models/survey_type.py +++ b/onadata/apps/logger/models/survey_type.py @@ -3,7 +3,7 @@ Survey type model class """ from django.db import models -from django.utils.encoding import python_2_unicode_compatible +from six import python_2_unicode_compatible @python_2_unicode_compatible @@ -11,10 +11,11 @@ class SurveyType(models.Model): """ Survey type model class """ + slug = models.CharField(max_length=100, unique=True) class Meta: - app_label = 'logger' + app_label = "logger" def __str__(self): return "SurveyType: %s" % self.slug diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index ea8b84296f..6f3503fa64 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -17,57 +17,71 @@ from django.db.models.signals import post_delete, post_save, pre_save from django.urls import reverse from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from future.utils import iteritems -from future.utils import listvalues +from six import iteritems, itervalues, python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase -from past.builtins import cmp -from pyxform import ( - SurveyElementBuilder, constants, create_survey_element_from_dict) +from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question from pyxform.section import RepeatingSection from pyxform.xform2json import create_survey_element_from_xml from taggit.managers import TaggableManager -from onadata.apps.logger.xform_instance_parser import (XLSFormError, - clean_and_parse_xml) +from onadata.apps.logger.xform_instance_parser import XLSFormError, clean_and_parse_xml from django.utils.html import conditional_escape from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.cache_tools import ( - IS_ORG, PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, - PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, XFORM_COUNT, - PROJ_OWNER_CACHE, XFORM_SUBMISSION_COUNT_FOR_DAY, - XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete) -from onadata.libs.utils.common_tags import (DURATION, ID, KNOWN_MEDIA_TYPES, - MEDIA_ALL_RECEIVED, MEDIA_COUNT, - NOTES, SUBMISSION_TIME, - SUBMITTED_BY, TAGS, TOTAL_MEDIA, - UUID, VERSION, REVIEW_STATUS, - REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, - DATE_MODIFIED) + IS_ORG, + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + XFORM_COUNT, + PROJ_OWNER_CACHE, + XFORM_SUBMISSION_COUNT_FOR_DAY, + XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, + safe_delete, +) +from onadata.libs.utils.common_tags import ( + DURATION, + ID, + KNOWN_MEDIA_TYPES, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + NOTES, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + REVIEW_STATUS, + REVIEW_COMMENT, + MULTIPLE_SELECT_TYPE, + DATE_MODIFIED, +) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _encode_for_mongo QUESTION_TYPES_TO_EXCLUDE = [ - u'note', + "note", ] XFORM_TITLE_LENGTH = 255 title_pattern = re.compile(r"(.*?)") +cmp = lambda x, y: (x > y) - (x < y) + + def question_types_to_exclude(_type): return _type in QUESTION_TYPES_TO_EXCLUDE def upload_to(instance, filename): - return os.path.join(instance.user.username, 'xls', - os.path.split(filename)[1]) + return os.path.join(instance.user.username, "xls", os.path.split(filename)[1]) -def contains_xml_invalid_char(text, invalids=['&', '>', '<']): +def contains_xml_invalid_char(text, invalids=["&", ">", "<"]): """Check whether 'text' contains ANY invalid xml chars""" return 1 in [c in text for c in invalids] @@ -79,21 +93,22 @@ def set_dict_iterator(self, dict_iterator): # Every section will get its own table # I need to think of an easy way to flatten out a dictionary # parent name, index, table name, data - def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, - parent_index): + def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_index): if table_name not in obs: obs[table_name] = [] this_index = len(obs[table_name]) - obs[table_name].append({ - u"_parent_table_name": parent_table_name, - u"_parent_index": parent_index, - }) + obs[table_name].append( + { + "_parent_table_name": parent_table_name, + "_parent_index": parent_index, + } + ) for (k, v) in iteritems(d): if isinstance(v, dict) and isinstance(v, list): if k in obs[table_name][-1]: raise AssertionError() obs[table_name][-1][k] = v - obs[table_name][-1][u"_index"] = this_index + obs[table_name][-1]["_index"] = this_index for (k, v) in iteritems(d): if isinstance(v, dict): @@ -102,7 +117,7 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, "obs": obs, "table_name": k, "parent_table_name": table_name, - "parent_index": this_index + "parent_index": this_index, } self._build_obs_from_dict(**kwargs) elif isinstance(v, list): @@ -125,7 +140,7 @@ def get_observation_from_dict(self, d): "d": d[root_name], "obs": {}, "table_name": root_name, - "parent_table_name": u"", + "parent_table_name": "", "parent_index": -1, } @@ -142,10 +157,13 @@ def get_forms_shared_with_user(user): """ xforms = XForm.objects.filter( pk__in=user.xformuserobjectpermission_set.values_list( - 'content_object_id', flat=True).distinct(), - downloadable=True, deleted_at__isnull=True) + "content_object_id", flat=True + ).distinct(), + downloadable=True, + deleted_at__isnull=True, + ) - return xforms.exclude(user=user).select_related('user') + return xforms.exclude(user=user).select_related("user") def check_version_set(survey): @@ -158,17 +176,14 @@ def check_version_set(survey): survey_json = json.loads(survey.to_json()) if not survey_json.get("version"): # set utc time as the default version - survey_json['version'] = \ - datetime.utcnow().strftime("%Y%m%d%H%M") + survey_json["version"] = datetime.utcnow().strftime("%Y%m%d%H%M") builder = SurveyElementBuilder() - survey = builder.create_survey_element_from_json( - json.dumps(survey_json)) + survey = builder.create_survey_element_from_json(json.dumps(survey_json)) return survey def _expand_select_all_that_apply(d, key, e): - if e and e.bind.get(u"type") == u"string"\ - and e.type == MULTIPLE_SELECT_TYPE: + if e and e.bind.get("type") == "string" and e.type == MULTIPLE_SELECT_TYPE: options_selected = d[key].split() for child in e.children: new_key = child.get_abbreviated_xpath() @@ -179,9 +194,9 @@ def _expand_select_all_that_apply(d, key, e): class XFormMixin(object): - GEODATA_SUFFIXES = ['latitude', 'longitude', 'altitude', 'precision'] + GEODATA_SUFFIXES = ["latitude", "longitude", "altitude", "precision"] - PREFIX_NAME_REGEX = re.compile(r'(?P.+/)(?P[^/]+)$') + PREFIX_NAME_REGEX = re.compile(r"(?P.+/)(?P[^/]+)$") def _set_uuid_in_xml(self, file_name=None): """ @@ -194,49 +209,55 @@ def _set_uuid_in_xml(self, file_name=None): doc = clean_and_parse_xml(self.xml) model_nodes = doc.getElementsByTagName("model") if len(model_nodes) != 1: - raise Exception(u"xml contains multiple model nodes") + raise Exception("xml contains multiple model nodes") model_node = model_nodes[0] instance_nodes = [ - node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and not node.hasAttribute("id") + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") ] if len(instance_nodes) != 1: - raise Exception(u"Multiple instance nodes without the id " - u"attribute, can't tell which is the main one") + raise Exception( + "Multiple instance nodes without the id " + "attribute, can't tell which is the main one" + ) instance_node = instance_nodes[0] # get the first child whose id attribute matches our id_string survey_nodes = [ - node for node in instance_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - (node.tagName == file_name or node.attributes.get('id')) + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and (node.tagName == file_name or node.attributes.get("id")) ] if len(survey_nodes) != 1: - raise Exception( - u"Multiple survey nodes with the id '%s'" % self.id_string) + raise Exception("Multiple survey nodes with the id '%s'" % self.id_string) survey_node = survey_nodes[0] formhub_nodes = [ - n for n in survey_node.childNodes + n + for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "formhub" ] if len(formhub_nodes) > 1: - raise Exception( - u"Multiple formhub nodes within main instance node") + raise Exception("Multiple formhub nodes within main instance node") elif len(formhub_nodes) == 1: formhub_node = formhub_nodes[0] else: formhub_node = survey_node.insertBefore( - doc.createElement("formhub"), survey_node.firstChild) + doc.createElement("formhub"), survey_node.firstChild + ) uuid_nodes = [ - node for node in formhub_node.childNodes + node + for node in formhub_node.childNodes if node.nodeType == Node.ELEMENT_NODE and node.tagName == "uuid" ] @@ -246,22 +267,25 @@ def _set_uuid_in_xml(self, file_name=None): # append the calculate bind node calculate_node = doc.createElement("bind") calculate_node.setAttribute( - "nodeset", "/%s/formhub/uuid" % survey_node.tagName) + "nodeset", "/%s/formhub/uuid" % survey_node.tagName + ) calculate_node.setAttribute("type", "string") calculate_node.setAttribute("calculate", "'%s'" % self.uuid) model_node.appendChild(calculate_node) - self.xml = doc.toprettyxml(indent=" ", encoding='utf-8') + self.xml = doc.toprettyxml(indent=" ", encoding="utf-8") # hack # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ - text_re = re.compile('(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*') - pretty_xml = text_re.sub(lambda m: ''.join(m.group(1, 2, 3)), - self.xml.decode('utf-8')) - inline_output = output_re.sub('\g<1>', pretty_xml) # noqa - inline_output = re.compile('').sub( # noqa - '', inline_output) + text_re = re.compile("(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*") + pretty_xml = text_re.sub( + lambda m: "".join(m.group(1, 2, 3)), self.xml.decode("utf-8") + ) + inline_output = output_re.sub("\g<1>", pretty_xml) # noqa + inline_output = re.compile("").sub( # noqa + "", inline_output + ) self.xml = inline_output class Meta: @@ -270,7 +294,7 @@ class Meta: @property def has_id_string_changed(self): - return getattr(self, '_id_string_changed', False) + return getattr(self, "_id_string_changed", False) def add_instances(self): _get_observation_from_dict = DictOrganizer().get_observation_from_dict @@ -293,8 +317,8 @@ def get_unique_id_string(self, id_string, count=0): # id_string already existed if self._id_string_already_exists_in_account(id_string): if count != 0: - if re.match(r'\w+_\d+$', id_string): - a = id_string.split('_') + if re.match(r"\w+_\d+$", id_string): + a = id_string.split("_") id_string = "_".join(a[:-1]) count += 1 id_string = "{}_{}".format(id_string, count) @@ -307,10 +331,9 @@ def get_survey(self): if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() - self._survey = \ - builder.create_survey_element_from_json(self.json) + self._survey = builder.create_survey_element_from_json(self.json) except ValueError: - xml = b(bytearray(self.xml, encoding='utf-8')) + xml = b(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) return self._survey @@ -331,8 +354,7 @@ def get_survey_element(self, name_or_xpath): # search by name if xpath fails fields = [ - field for field in self.get_survey_elements() - if field.name == name_or_xpath + field for field in self.get_survey_elements() if field.name == name_or_xpath ] return fields[0] if len(fields) else None @@ -343,16 +365,17 @@ def get_child_elements(self, name_or_xpath, split_select_multiples=True): appended to the list. If the name_or_xpath is a repeat we iterate through the child elements as well. """ - GROUP_AND_SELECT_MULTIPLES = ['group'] + GROUP_AND_SELECT_MULTIPLES = ["group"] if split_select_multiples: - GROUP_AND_SELECT_MULTIPLES += ['select all that apply'] + GROUP_AND_SELECT_MULTIPLES += ["select all that apply"] def flatten(elem, items=[]): results = [] if elem: xpath = elem.get_abbreviated_xpath() - if elem.type in GROUP_AND_SELECT_MULTIPLES or \ - (xpath == name_or_xpath and elem.type == 'repeat'): + if elem.type in GROUP_AND_SELECT_MULTIPLES or ( + xpath == name_or_xpath and elem.type == "repeat" + ): for child in elem.children: results += flatten(child) else: @@ -364,16 +387,14 @@ def flatten(elem, items=[]): return flatten(element) - def get_choice_label(self, field, choice_value, lang='English'): - choices = [ - choice for choice in field.children if choice.name == choice_value - ] + def get_choice_label(self, field, choice_value, lang="English"): + choices = [choice for choice in field.children if choice.name == choice_value] if len(choices): choice = choices[0] label = choice.label if isinstance(label, dict): - label = label.get(lang, listvalues(choice.label)[0]) + label = label.get(lang, itervalues(choice.label)[0]) return label @@ -386,24 +407,27 @@ def get_mongo_field_names_dict(self): """ names = {} for elem in self.get_survey_elements(): - names[_encode_for_mongo(text(elem.get_abbreviated_xpath()))] = \ - elem.get_abbreviated_xpath() + names[ + _encode_for_mongo(text(elem.get_abbreviated_xpath())) + ] = elem.get_abbreviated_xpath() return names survey_elements = property(get_survey_elements) def get_field_name_xpaths_only(self): return [ - elem.get_abbreviated_xpath() for elem in self.survey_elements - if elem.type != '' and elem.type != 'survey' + elem.get_abbreviated_xpath() + for elem in self.survey_elements + if elem.type != "" and elem.type != "survey" ] def geopoint_xpaths(self): survey_elements = self.get_survey_elements() return [ - e.get_abbreviated_xpath() for e in survey_elements - if e.bind.get(u'type') == u'geopoint' + e.get_abbreviated_xpath() + for e in survey_elements + if e.bind.get("type") == "geopoint" ] def xpath_of_first_geopoint(self): @@ -411,11 +435,7 @@ def xpath_of_first_geopoint(self): return len(geo_xpaths) and geo_xpaths[0] - def xpaths(self, - prefix='', - survey_element=None, - result=None, - repeat_iterations=4): + def xpaths(self, prefix="", survey_element=None, result=None, repeat_iterations=4): """ Return a list of XPaths for this survey that will be used as headers for the csv export. @@ -426,13 +446,15 @@ def xpaths(self, return [] result = [] if result is None else result - path = '/'.join([prefix, text(survey_element.name)]) + path = "/".join([prefix, text(survey_element.name)]) if survey_element.children is not None: # add xpaths to result for each child - indices = [''] if not isinstance(survey_element, - RepeatingSection) else \ - ['[%d]' % (i + 1) for i in range(repeat_iterations)] + indices = ( + [""] + if not isinstance(survey_element, RepeatingSection) + else ["[%d]" % (i + 1) for i in range(repeat_iterations)] + ) for i in indices: for e in survey_element.children: self.xpaths(path + i, e, result, repeat_iterations) @@ -442,12 +464,14 @@ def xpaths(self, # replace the single question column with a column for each # item in a select all that apply question. - if survey_element.bind.get(u'type') == u'string' \ - and survey_element.type == MULTIPLE_SELECT_TYPE: + if ( + survey_element.bind.get("type") == "string" + and survey_element.type == MULTIPLE_SELECT_TYPE + ): result.pop() for child in survey_element.children: - result.append('/'.join([path, child.name])) - elif survey_element.bind.get(u'type') == u'geopoint': + result.append("/".join([path, child.name])) + elif survey_element.bind.get("type") == "geopoint": result += self.get_additional_geopoint_xpaths(path) return result @@ -461,39 +485,50 @@ def get_additional_geopoint_xpaths(cls, xpath): DataDictionary.GEODATA_SUFFIXES """ match = cls.PREFIX_NAME_REGEX.match(xpath) - prefix = '' + prefix = "" name = xpath if match: - prefix = match.groupdict()['prefix'] - name = match.groupdict()['name'] + prefix = match.groupdict()["prefix"] + name = match.groupdict()["name"] - return [ - '_'.join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES - ] + return ["_".join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES] def _additional_headers(self): return [ - u'_xform_id_string', u'_percentage_complete', u'_status', - u'_attachments', u'_potential_duplicates' + "_xform_id_string", + "_percentage_complete", + "_status", + "_attachments", + "_potential_duplicates", ] - def get_headers( - self, include_additional_headers=False, repeat_iterations=4): + def get_headers(self, include_additional_headers=False, repeat_iterations=4): """ Return a list of headers for a csv file. """ + def shorten(xpath): - xpath_list = xpath.split('/') - return '/'.join(xpath_list[2:]) + xpath_list = xpath.split("/") + return "/".join(xpath_list[2:]) header_list = [ - shorten(xpath) for xpath in self.xpaths( - repeat_iterations=repeat_iterations)] + shorten(xpath) for xpath in self.xpaths(repeat_iterations=repeat_iterations) + ] header_list += [ - ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES, - REVIEW_STATUS, REVIEW_COMMENT, VERSION, DURATION, - SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT, - MEDIA_ALL_RECEIVED + ID, + UUID, + SUBMISSION_TIME, + DATE_MODIFIED, + TAGS, + NOTES, + REVIEW_STATUS, + REVIEW_COMMENT, + VERSION, + DURATION, + SUBMITTED_BY, + TOTAL_MEDIA, + MEDIA_COUNT, + MEDIA_ALL_RECEIVED, ] if include_additional_headers: header_list += self._additional_headers() @@ -501,7 +536,7 @@ def shorten(xpath): def get_keys(self): def remove_first_index(xpath): - return re.sub(r'\[1\]', '', xpath) + return re.sub(r"\[1\]", "", xpath) return [remove_first_index(header) for header in self.get_headers()] @@ -512,15 +547,14 @@ def get_element(self, abbreviated_xpath): self._survey_elements[e.get_abbreviated_xpath()] = e def remove_all_indices(xpath): - return re.sub(r"\[\d+\]", u"", xpath) + return re.sub(r"\[\d+\]", "", xpath) clean_xpath = remove_all_indices(abbreviated_xpath) return self._survey_elements.get(clean_xpath) def get_default_language(self): - if not hasattr(self, '_default_language'): - self._default_language = \ - self.survey.to_json_dict().get('default_language') + if not hasattr(self, "_default_language"): + self._default_language = self.survey.to_json_dict().get("default_language") return self._default_language @@ -547,21 +581,19 @@ def get_label(self, abbreviated_xpath, elem=None, language=None): label = label[language] else: language = self.get_language(list(label)) - label = label[language] if language else '' + label = label[language] if language else "" return label def get_xpath_cmp(self): if not hasattr(self, "_xpaths"): - self._xpaths = [ - e.get_abbreviated_xpath() for e in self.survey_elements - ] + self._xpaths = [e.get_abbreviated_xpath() for e in self.survey_elements] def xpath_cmp(x, y): # For the moment, we aren't going to worry about repeating # nodes. - new_x = re.sub(r"\[\d+\]", u"", x) - new_y = re.sub(r"\[\d+\]", u"", y) + new_x = re.sub(r"\[\d+\]", "", x) + new_y = re.sub(r"\[\d+\]", "", y) if new_x == new_y: return cmp(x, y) if new_x not in self._xpaths and new_y not in self._xpaths: @@ -612,7 +644,7 @@ def _rename_key(self, d, old_key, new_key): del d[old_key] def _expand_geocodes(self, d, key, e): - if e and e.bind.get(u"type") == u"geopoint": + if e and e.bind.get("type") == "geopoint": geodata = d[key].split() for i in range(len(geodata)): new_key = "%s_%s" % (key, self.geodata_suffixes[i]) @@ -634,15 +666,11 @@ def _mark_start_time_boolean(self): self.has_start_time = False def get_survey_elements_of_type(self, element_type): - return [ - e for e in self.get_survey_elements() if e.type == element_type - ] + return [e for e in self.get_survey_elements() if e.type == element_type] def get_survey_elements_with_choices(self): - if not hasattr(self, '_survey_elements_with_choices'): - choices_type = [ - constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY - ] + if not hasattr(self, "_survey_elements_with_choices"): + choices_type = [constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY] self._survey_elements_with_choices = [ e for e in self.get_survey_elements() if e.type in choices_type @@ -654,11 +682,17 @@ def get_select_one_xpaths(self): """ Returns abbreviated_xpath for SELECT_ONE questions in the survey. """ - if not hasattr(self, '_select_one_xpaths'): + if not hasattr(self, "_select_one_xpaths"): self._select_one_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ONE]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ONE] + ], + [], + ) + ] return self._select_one_xpaths @@ -667,20 +701,26 @@ def get_select_multiple_xpaths(self): Returns abbreviated_xpath for SELECT_ALL_THAT_APPLY questions in the survey. """ - if not hasattr(self, '_select_multiple_xpaths'): + if not hasattr(self, "_select_multiple_xpaths"): self._select_multiple_xpaths = [ - e.get_abbreviated_xpath() for e in sum([ - self.get_survey_elements_of_type(select) - for select in [constants.SELECT_ALL_THAT_APPLY]], [])] + e.get_abbreviated_xpath() + for e in sum( + [ + self.get_survey_elements_of_type(select) + for select in [constants.SELECT_ALL_THAT_APPLY] + ], + [], + ) + ] return self._select_multiple_xpaths def get_media_survey_xpaths(self): return [ e.get_abbreviated_xpath() - for e in sum([ - self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES - ], []) + for e in sum( + [self.get_survey_elements_of_type(m) for m in KNOWN_MEDIA_TYPES], [] + ) ] def get_osm_survey_xpaths(self): @@ -689,93 +729,102 @@ def get_osm_survey_xpaths(self): """ return [ elem.get_abbreviated_xpath() - for elem in self.get_survey_elements_of_type('osm')] + for elem in self.get_survey_elements_of_type("osm") + ] @python_2_unicode_compatible class XForm(XFormMixin, BaseModel): - CLONED_SUFFIX = '_cloned' + CLONED_SUFFIX = "_cloned" MAX_ID_LENGTH = 100 xls = models.FileField(upload_to=upload_to, null=True) - json = models.TextField(default=u'') - description = models.TextField(default=u'', null=True, blank=True) + json = models.TextField(default="") + description = models.TextField(default="", null=True, blank=True) xml = models.TextField() user = models.ForeignKey( - User, related_name='xforms', null=True, on_delete=models.CASCADE) + User, related_name="xforms", null=True, on_delete=models.CASCADE + ) require_auth = models.BooleanField(default=False) shared = models.BooleanField(default=False) shared_data = models.BooleanField(default=False) downloadable = models.BooleanField(default=True) allows_sms = models.BooleanField(default=False) encrypted = models.BooleanField(default=False) - deleted_by = models.ForeignKey(User, related_name='xform_deleted_by', - null=True, on_delete=models.SET_NULL, - default=None, blank=True) + deleted_by = models.ForeignKey( + User, + related_name="xform_deleted_by", + null=True, + on_delete=models.SET_NULL, + default=None, + blank=True, + ) # the following fields are filled in automatically sms_id_string = models.SlugField( editable=False, verbose_name=ugettext_lazy("SMS ID"), max_length=MAX_ID_LENGTH, - default='') + default="", + ) id_string = models.SlugField( - editable=False, - verbose_name=ugettext_lazy("ID"), - max_length=MAX_ID_LENGTH) + editable=False, verbose_name=ugettext_lazy("ID"), max_length=MAX_ID_LENGTH + ) title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(blank=True, null=True) last_submission_time = models.DateTimeField(blank=True, null=True) has_start_time = models.BooleanField(default=False) - uuid = models.CharField(max_length=36, default=u'', db_index=True) - public_key = models.TextField(default='', blank=True, null=True) + uuid = models.CharField(max_length=36, default="", db_index=True) + public_key = models.TextField(default="", blank=True, null=True) - uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', - re.DOTALL) - instance_id_regex = re.compile(r'.*?id="([^"]+)".*', - re.DOTALL) + uuid_regex = re.compile(r'(.*?id="[^"]+">)(.*)(.*)', re.DOTALL) + instance_id_regex = re.compile(r'.*?id="([^"]+)".*', re.DOTALL) uuid_node_location = 2 uuid_bind_location = 4 - bamboo_dataset = models.CharField(max_length=60, default=u'') + bamboo_dataset = models.CharField(max_length=60, default="") instances_with_geopoints = models.BooleanField(default=False) instances_with_osm = models.BooleanField(default=False) num_of_submissions = models.IntegerField(default=0) - version = models.CharField( - max_length=XFORM_TITLE_LENGTH, null=True, blank=True) - project = models.ForeignKey('Project', on_delete=models.CASCADE) + version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True, blank=True) + project = models.ForeignKey("Project", on_delete=models.CASCADE) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) metadata_set = GenericRelation( - 'main.MetaData', - content_type_field='content_type_id', - object_id_field="object_id") + "main.MetaData", + content_type_field="content_type_id", + object_id_field="object_id", + ) has_hxl_support = models.BooleanField(default=False) last_updated_at = models.DateTimeField(auto_now=True) - hash = models.CharField(_("Hash"), max_length=36, blank=True, null=True, - default=None) + hash = models.CharField( + _("Hash"), max_length=36, blank=True, null=True, default=None + ) # XForm was created as a merged dataset is_merged_dataset = models.BooleanField(default=False) tags = TaggableManager() class Meta: - app_label = 'logger' - unique_together = (("user", "id_string", "project"), - ("user", "sms_id_string", "project")) + app_label = "logger" + unique_together = ( + ("user", "id_string", "project"), + ("user", "sms_id_string", "project"), + ) verbose_name = ugettext_lazy("XForm") verbose_name_plural = ugettext_lazy("XForms") - ordering = ("pk", ) + ordering = ("pk",) permissions = ( ("view_xform_all", _("Can view all associated data")), ("view_xform_data", _("Can view submitted data")), ("report_xform", _("Can make submissions to the form")), - ("move_xform", _(u"Can move form between projects")), - ("transfer_xform", _(u"Can transfer form ownership.")), - ("can_export_xform_data", _(u"Can export form data")), - ("delete_submission", _(u"Can delete submissions from form")), + ("move_xform", _("Can move form between projects")), + ("transfer_xform", _("Can transfer form ownership.")), + ("can_export_xform_data", _("Can export form data")), + ("delete_submission", _("Can delete submissions from form")), ) def file_name(self): @@ -784,10 +833,8 @@ def file_name(self): def url(self): return reverse( "download_xform", - kwargs={ - "username": self.user.username, - "id_string": self.id_string - }) + kwargs={"username": self.user.username, "id_string": self.id_string}, + ) @property def has_instances_with_geopoints(self): @@ -809,24 +856,25 @@ def _set_title(self): if matches: title_xml = matches[0][:XFORM_TITLE_LENGTH] else: - title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else '' + title_xml = self.title[:XFORM_TITLE_LENGTH] if self.title else "" if self.title and title_xml != self.title: title_xml = self.title[:XFORM_TITLE_LENGTH] if isinstance(self.xml, b): - self.xml = self.xml.decode('utf-8') - self.xml = title_pattern.sub(u"%s" % title_xml, - self.xml) + self.xml = self.xml.decode("utf-8") + self.xml = title_pattern.sub("%s" % title_xml, self.xml) self._set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( - _("Title shouldn't have any invalid xml " - "characters ('>' '&' '<')")) + _("Title shouldn't have any invalid xml " "characters ('>' '&' '<')") + ) # Capture urls within form title - if re.search(r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", self.title): # noqa - raise XLSFormError( - _("Invalid title value; value shouldn't match a URL")) + if re.search( + r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", + self.title, + ): # noqa + raise XLSFormError(_("Invalid title value; value shouldn't match a URL")) self.title = title_xml @@ -834,15 +882,15 @@ def _set_hash(self): self.hash = self.get_hash() def _set_encrypted_field(self): - if self.json and self.json != '': + if self.json and self.json != "": json_dict = json.loads(self.json) - self.encrypted = 'public_key' in json_dict + self.encrypted = "public_key" in json_dict def _set_public_key_field(self): - if self.json and self.json != '': + if self.json and self.json != "": if self.num_of_submissions == 0 and self.public_key: json_dict = json.loads(self.json) - json_dict['public_key'] = self.public_key + json_dict["public_key"] = self.public_key survey = create_survey_element_from_dict(json_dict) self.json = survey.to_json() self.xml = survey.to_xml() @@ -852,71 +900,84 @@ def update(self, *args, **kwargs): super(XForm, self).save(*args, **kwargs) def save(self, *args, **kwargs): - update_fields = kwargs.get('update_fields') + update_fields = kwargs.get("update_fields") if update_fields: - kwargs['update_fields'] = list( - set(list(update_fields) + ['date_modified'])) - if update_fields is None or 'title' in update_fields: + kwargs["update_fields"] = list(set(list(update_fields) + ["date_modified"])) + if update_fields is None or "title" in update_fields: self._set_title() if self.pk is None: self._set_hash() - if update_fields is None or 'encrypted' in update_fields: + if update_fields is None or "encrypted" in update_fields: self._set_encrypted_field() - if update_fields is None or 'id_string' in update_fields: + if update_fields is None or "id_string" in update_fields: old_id_string = self.id_string if not self.deleted_at: self._set_id_string() # check if we have an existing id_string, # if so, the one must match but only if xform is NOT new - if self.pk and old_id_string and old_id_string != self.id_string \ - and self.num_of_submissions > 0: + if ( + self.pk + and old_id_string + and old_id_string != self.id_string + and self.num_of_submissions > 0 + ): raise XLSFormError( - _(u"Your updated form's id_string '%(new_id)s' must match " - "the existing forms' id_string '%(old_id)s'." % - {'new_id': self.id_string, - 'old_id': old_id_string})) - - if getattr(settings, 'STRICT', True) and \ - not re.search(r"^[\w-]+$", self.id_string): + _( + "Your updated form's id_string '%(new_id)s' must match " + "the existing forms' id_string '%(old_id)s'." + % {"new_id": self.id_string, "old_id": old_id_string} + ) + ) + + if getattr(settings, "STRICT", True) and not re.search( + r"^[\w-]+$", self.id_string + ): raise XLSFormError( - _(u'In strict mode, the XForm ID must be a ' - 'valid slug and contain no spaces. Please ensure' - ' that you have set an id_string in the settings sheet ' - 'or have modified the filename to not contain' - ' any spaces.')) - - if not self.sms_id_string and (update_fields is None or - 'id_string' in update_fields): + _( + "In strict mode, the XForm ID must be a " + "valid slug and contain no spaces. Please ensure" + " that you have set an id_string in the settings sheet " + "or have modified the filename to not contain" + " any spaces." + ) + ) + + if not self.sms_id_string and ( + update_fields is None or "id_string" in update_fields + ): try: # try to guess the form's wanted sms_id_string # from it's json rep (from XLSForm) # otherwise, use id_string to ensure uniqueness self.sms_id_string = json.loads(self.json).get( - 'sms_keyword', self.id_string) + "sms_keyword", self.id_string + ) except Exception: self.sms_id_string = self.id_string - if update_fields is None or 'public_key' in update_fields: + if update_fields is None or "public_key" in update_fields: self._set_public_key_field() - if 'skip_xls_read' in kwargs: - del kwargs['skip_xls_read'] + if "skip_xls_read" in kwargs: + del kwargs["skip_xls_read"] - if (self.id_string and len( - self.id_string) > self.MAX_ID_LENGTH) or \ - (self.sms_id_string and len( - self.sms_id_string) > self.MAX_ID_LENGTH): + if (self.id_string and len(self.id_string) > self.MAX_ID_LENGTH) or ( + self.sms_id_string and len(self.sms_id_string) > self.MAX_ID_LENGTH + ): raise XLSFormError( - _(u'The XForm id_string provided exceeds %s characters.' - ' Please change the "id_string" or "form_id" values' - 'in settings sheet or reduce the file name if you do' - ' not have a settings sheets.' % self.MAX_ID_LENGTH)) + _( + "The XForm id_string provided exceeds %s characters." + ' Please change the "id_string" or "form_id" values' + "in settings sheet or reduce the file name if you do" + " not have a settings sheets." % self.MAX_ID_LENGTH + ) + ) is_version_available = self.version is not None if is_version_available and contains_xml_invalid_char(self.version): raise XLSFormError( - _("Version shouldn't have any invalid " - "characters ('>' '&' '<')")) + _("Version shouldn't have any invalid " "characters ('>' '&' '<')") + ) self.description = conditional_escape(self.description) @@ -936,49 +997,56 @@ def soft_delete(self, user=None): """ soft_deletion_time = timezone.now() - deletion_suffix = soft_deletion_time.strftime('-deleted-at-%s') + deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time self.id_string += deletion_suffix self.sms_id_string += deletion_suffix self.downloadable = False # only take the first 100 characters (within the set max_length) - self.id_string = self.id_string[:self.MAX_ID_LENGTH] - self.sms_id_string = self.sms_id_string[:self.MAX_ID_LENGTH] - - update_fields = ['date_modified', 'deleted_at', 'id_string', - 'sms_id_string', 'downloadable'] + self.id_string = self.id_string[: self.MAX_ID_LENGTH] + self.sms_id_string = self.sms_id_string[: self.MAX_ID_LENGTH] + + update_fields = [ + "date_modified", + "deleted_at", + "id_string", + "sms_id_string", + "downloadable", + ] if user is not None: self.deleted_by = user - update_fields.append('deleted_by') + update_fields.append("deleted_by") self.save(update_fields=update_fields) # Delete associated filtered datasets for dataview in self.dataview_set.all(): dataview.soft_delete(user) # Delete associated Merged-Datasets - for merged_dataset in self.mergedxform_ptr.filter( - deleted_at__isnull=True): + for merged_dataset in self.mergedxform_ptr.filter(deleted_at__isnull=True): merged_dataset.soft_delete(user) # Delete associated Form Media Files - for metadata in self.metadata_set.filter( - deleted_at__isnull=True): + for metadata in self.metadata_set.filter(deleted_at__isnull=True): metadata.soft_delete() def submission_count(self, force_update=False): if self.num_of_submissions == 0 or force_update: if self.is_merged_dataset: - count = self.mergedxform.xforms.aggregate( - num=Sum('num_of_submissions')).get('num') or 0 + count = ( + self.mergedxform.xforms.aggregate( + num=Sum("num_of_submissions") + ).get("num") + or 0 + ) else: count = self.instances.filter(deleted_at__isnull=True).count() if count != self.num_of_submissions: self.num_of_submissions = count - self.save(update_fields=['num_of_submissions']) + self.save(update_fields=["num_of_submissions"]) # clear cache - key = '{}{}'.format(XFORM_COUNT, self.pk) + key = "{}{}".format(XFORM_COUNT, self.pk) safe_delete(key) return self.num_of_submissions @@ -991,23 +1059,28 @@ def submission_count_for_today(self): current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() current_date = current_timezone.localize( - datetime(today.year, today.month, today.day)).isoformat() - count = cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") if cache.get( - f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}" - ) == current_date else 0 + datetime(today.year, today.month, today.day) + ).isoformat() + count = ( + cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") + if cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}") + == current_date + else 0 + ) return count def geocoded_submission_count(self): """Number of geocoded submissions.""" return self.instances.filter( - deleted_at__isnull=True, geom__isnull=False).count() + deleted_at__isnull=True, geom__isnull=False + ).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: try: - last_submission = self.instances.\ - filter(deleted_at__isnull=True).latest("date_created") + last_submission = self.instances.filter(deleted_at__isnull=True).latest( + "date_created" + ) except ObjectDoesNotExist: pass else: @@ -1023,7 +1096,7 @@ def time_of_last_submission_update(self): pass def get_hash(self): - return u'md5:%s' % md5(self.xml.encode('utf8')).hexdigest() + return "md5:%s" % md5(self.xml.encode("utf8")).hexdigest() @property def can_be_replaced(self): @@ -1037,8 +1110,7 @@ def public_forms(cls): def update_profile_num_submissions(sender, instance, **kwargs): profile_qs = User.profile.get_queryset() try: - profile = profile_qs.select_for_update()\ - .get(pk=instance.user.profile.pk) + profile = profile_qs.select_for_update().get(pk=instance.user.profile.pk) except ObjectDoesNotExist: pass else: @@ -1051,15 +1123,16 @@ def update_profile_num_submissions(sender, instance, **kwargs): post_delete.connect( update_profile_num_submissions, sender=XForm, - dispatch_uid='update_profile_num_submissions') + dispatch_uid="update_profile_num_submissions", +) def clear_project_cache(project_id): - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_BASE_FORMS_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_SUB_DATE_CACHE, project_id)) - safe_delete('{}{}'.format(PROJ_NUM_DATASET_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_OWNER_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_FORMS_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, project_id)) + safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, project_id)) def set_object_permissions(sender, instance=None, created=False, **kwargs): @@ -1067,30 +1140,31 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): project = instance.project project.refresh_from_db() clear_project_cache(project.pk) - safe_delete('{}{}'.format(IS_ORG, instance.pk)) + safe_delete("{}{}".format(IS_ORG, instance.pk)) if created: from onadata.libs.permissions import OwnerRole + OwnerRole.add(instance.user, instance) if instance.created_by and instance.user != instance.created_by: OwnerRole.add(instance.created_by, instance) from onadata.libs.utils.project_utils import set_project_perms_to_xform + set_project_perms_to_xform(instance, project) post_save.connect( - set_object_permissions, - sender=XForm, - dispatch_uid='xform_object_permissions') + set_object_permissions, sender=XForm, dispatch_uid="xform_object_permissions" +) def save_project(sender, instance=None, created=False, **kwargs): - instance.project.save(update_fields=['date_modified']) + instance.project.save(update_fields=["date_modified"]) -pre_save.connect(save_project, sender=XForm, dispatch_uid='save_project_xform') +pre_save.connect(save_project, sender=XForm, dispatch_uid="save_project_xform") def xform_post_delete_callback(sender, instance, **kwargs): @@ -1099,9 +1173,8 @@ def xform_post_delete_callback(sender, instance, **kwargs): post_delete.connect( - xform_post_delete_callback, - sender=XForm, - dispatch_uid='xform_post_delete_callback') + xform_post_delete_callback, sender=XForm, dispatch_uid="xform_post_delete_callback" +) class XFormUserObjectPermission(UserObjectPermissionBase): @@ -1121,12 +1194,10 @@ def check_xform_uuid(new_uuid): Checks if a new_uuid has already been used, if it has it raises the exception DuplicateUUIDError. """ - count = XForm.objects.filter(uuid=new_uuid, - deleted_at__isnull=True).count() + count = XForm.objects.filter(uuid=new_uuid, deleted_at__isnull=True).count() if count > 0: - raise DuplicateUUIDError( - "An xform with uuid: %s already exists" % new_uuid) + raise DuplicateUUIDError("An xform with uuid: %s already exists" % new_uuid) def update_xform_uuid(username, id_string, new_uuid): diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 58f7cfcc57..8cb0596f70 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -10,7 +10,7 @@ from django.test import RequestFactory from django.urls import reverse from django_digest.test import Client as DigestClient -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from httmock import HTTMock, urlmatch from onadata.apps.logger.models import Instance, XForm @@ -23,28 +23,29 @@ storage = get_storage_class()() -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def form_list_xml(url, request, **kwargs): response = requests.Response() factory = RequestFactory() req = factory.get(url.path) - req.user = authenticate(username='bob', password='bob') + req.user = authenticate(username="bob", password="bob") req.user.profile.require_auth = False req.user.profile.save() - id_string = 'transportation_2011_07_25' - if url.path.endswith('formList'): - res = formList(req, username='bob') - elif url.path.endswith('form.xml'): - res = download_xform(req, username='bob', id_string=id_string) - elif url.path.find('xformsManifest') > -1: - res = xformsManifest(req, username='bob', id_string=id_string) - elif url.path.find('formid-media') > -1: - data_id = url.path[url.path.rfind('/') + 1:] + id_string = "transportation_2011_07_25" + if url.path.endswith("formList"): + res = formList(req, username="bob") + elif url.path.endswith("form.xml"): + res = download_xform(req, username="bob", id_string=id_string) + elif url.path.find("xformsManifest") > -1: + res = xformsManifest(req, username="bob", id_string=id_string) + elif url.path.find("formid-media") > -1: + data_id = url.path[url.path.rfind("/") + 1 :] res = download_media_data( - req, username='bob', id_string=id_string, data_id=data_id) + req, username="bob", id_string=id_string, data_id=data_id + ) response._content = get_streaming_content(res) else: - res = formList(req, username='bob') + res = formList(req, username="bob") response.status_code = 200 if not response._content: response._content = res.content @@ -60,15 +61,15 @@ def get_streaming_content(res): return content -@urlmatch(netloc=r'(.*\.)?testserver$') +@urlmatch(netloc=r"(.*\.)?testserver$") def instances_xml(url, request, **kwargs): response = requests.Response() client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') - res = client.get('%s?%s' % (url.path, url.query)) + client.set_authorization("bob", "bob", "Digest") + res = client.get("%s?%s" % (url.path, url.query)) if res.status_code == 302: - res = client.get(res['Location']) - response.encoding = res.get('content-type') + res = client.get(res["Location"]) + response.encoding = res.get("content-type") response._content = get_streaming_content(res) else: response._content = res.content @@ -77,27 +78,24 @@ def instances_xml(url, request, **kwargs): class TestBriefcaseClient(TestBase): - def setUp(self): TestBase.setUp(self) self._publish_transportation_form() self._submit_transport_instance_w_attachment() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + uf = UploadedFile(file=open(src, "rb"), content_type="image/png") count = MetaData.objects.count() MetaData.media_upload(self.xform, uf) self.assertEqual(MetaData.objects.count(), count + 1) url = urljoin( - self.base_url, - reverse(profile, kwargs={'username': self.user.username}) + self.base_url, reverse(profile, kwargs={"username": self.user.username}) ) self._logout() - self._create_user_and_login('deno', 'deno') + self._create_user_and_login("deno", "deno") self.bc = BriefcaseClient( - username='bob', password='bob', - url=url, - user=self.user + username="bob", password="bob", url=url, user=self.user ) def test_download_xform_xml(self): @@ -107,14 +105,14 @@ def test_download_xform_xml(self): with HTTMock(form_list_xml): self.bc.download_xforms() forms_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string) + "deno", "briefcase", "forms", self.xform.id_string + ) self.assertTrue(storage.exists(forms_folder_path)) - forms_path = os.path.join(forms_folder_path, - '%s.xml' % self.xform.id_string) + forms_path = os.path.join(forms_folder_path, "%s.xml" % self.xform.id_string) self.assertTrue(storage.exists(forms_path)) - form_media_path = os.path.join(forms_folder_path, 'form-media') + form_media_path = os.path.join(forms_folder_path, "form-media") self.assertTrue(storage.exists(form_media_path)) - media_path = os.path.join(form_media_path, 'screenshot.png') + media_path = os.path.join(form_media_path, "screenshot.png") self.assertTrue(storage.exists(media_path)) """ @@ -123,15 +121,18 @@ def test_download_xform_xml(self): with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) instance_folder_path = os.path.join( - 'deno', 'briefcase', 'forms', self.xform.id_string, 'instances') + "deno", "briefcase", "forms", self.xform.id_string, "instances" + ) self.assertTrue(storage.exists(instance_folder_path)) instance = Instance.objects.all()[0] instance_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, 'submission.xml') + instance_folder_path, "uuid%s" % instance.uuid, "submission.xml" + ) self.assertTrue(storage.exists(instance_path)) media_file = "1335783522563.jpg" media_path = os.path.join( - instance_folder_path, 'uuid%s' % instance.uuid, media_file) + instance_folder_path, "uuid%s" % instance.uuid, media_file + ) self.assertTrue(storage.exists(media_path)) def test_push(self): @@ -140,22 +141,22 @@ def test_push(self): with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) XForm.objects.all().delete() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 0) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 0) self.bc.push() - xforms = XForm.objects.filter( - user=self.user, id_string=self.xform.id_string) + xforms = XForm.objects.filter(user=self.user, id_string=self.xform.id_string) self.assertEqual(xforms.count(), 1) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=self.xform.id_string) + xform__user=self.user, xform__id_string=self.xform.id_string + ) self.assertEqual(instances.count(), 1) def tearDown(self): # remove media files - for username in ['bob', 'deno']: + for username in ["bob", "deno"]: if storage.exists(username): shutil.rmtree(storage.path(username)) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index 652ad42436..d6ceace3ce 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -2,7 +2,7 @@ import re import dateutil.parser from builtins import str as text -from future.utils import python_2_unicode_compatible +from six import python_2_unicode_compatible from xml.dom import minidom, Node from django.utils.encoding import smart_text, smart_str @@ -18,25 +18,25 @@ class XLSFormError(Exception): @python_2_unicode_compatible class DuplicateInstance(Exception): def __str__(self): - return _(u'Duplicate Instance') + return _("Duplicate Instance") @python_2_unicode_compatible class InstanceInvalidUserError(Exception): def __str__(self): - return _(u'Could not determine the user.') + return _("Could not determine the user.") @python_2_unicode_compatible class InstanceParseError(Exception): def __str__(self): - return _(u'The instance could not be parsed.') + return _("The instance could not be parsed.") @python_2_unicode_compatible class InstanceEmptyError(InstanceParseError): def __str__(self): - return _(u'Empty instance') + return _("Empty instance") class InstanceFormatError(Exception): @@ -67,25 +67,31 @@ def get_meta_from_xml(xml_str, meta_name): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - meta_tags = [n for n in survey_node.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == "meta" or - n.tagName.lower() == "orx:meta")] + meta_tags = [ + n + for n in survey_node.childNodes + if n.nodeType == Node.ELEMENT_NODE + and (n.tagName.lower() == "meta" or n.tagName.lower() == "orx:meta") + ] if len(meta_tags) == 0: return None # get the requested tag meta_tag = meta_tags[0] - uuid_tags = [n for n in meta_tag.childNodes if - n.nodeType == Node.ELEMENT_NODE and - (n.tagName.lower() == meta_name.lower() or - n.tagName.lower() == u'orx:%s' % meta_name.lower())] + uuid_tags = [ + n + for n in meta_tag.childNodes + if n.nodeType == Node.ELEMENT_NODE + and ( + n.tagName.lower() == meta_name.lower() + or n.tagName.lower() == "orx:%s" % meta_name.lower() + ) + ] if len(uuid_tags) == 0: return None uuid_tag = uuid_tags[0] - return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild\ - else None + return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild else None def get_uuid_from_xml(xml): @@ -94,6 +100,7 @@ def _uuid_only(uuid, regex): if matches and len(matches.groups()) > 0: return matches.groups()[0] return None + uuid = get_meta_from_xml(xml, "instanceID") regex = re.compile(r"uuid:(.*)") if uuid: @@ -106,8 +113,8 @@ def _uuid_only(uuid, regex): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - uuid = survey_node.getAttribute('instanceID') - if uuid != '': + uuid = survey_node.getAttribute("instanceID") + if uuid != "": return _uuid_only(uuid, regex) return None @@ -121,8 +128,8 @@ def get_submission_date_from_xml(xml): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - submissionDate = survey_node.getAttribute('submissionDate') - if submissionDate != '': + submissionDate = survey_node.getAttribute("submissionDate") + if submissionDate != "": return dateutil.parser.parse(submissionDate) return None @@ -139,7 +146,7 @@ def get_deprecated_uuid_from_xml(xml): def clean_and_parse_xml(xml_string): clean_xml_str = xml_string.strip() - clean_xml_str = re.sub(r">\s+<", u"><", smart_text(clean_xml_str)) + clean_xml_str = re.sub(r">\s+<", "><", smart_text(clean_xml_str)) xml_obj = minidom.parseString(smart_str(clean_xml_str)) return xml_obj @@ -148,8 +155,7 @@ def _xml_node_to_dict(node, repeats=[], encrypted=False): if len(node.childNodes) == 0: # there's no data for this leaf node return None - elif len(node.childNodes) == 1 and \ - node.childNodes[0].nodeType == node.TEXT_NODE: + elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node return {node.nodeName: node.childNodes[0].nodeValue} else: @@ -173,7 +179,7 @@ def _xml_node_to_dict(node, repeats=[], encrypted=False): node_type = dict # check if name is in list of repeats and make it a list if so # All the photo attachments in an encrypted form use name media - if child_xpath in repeats or (encrypted and child_name == 'media'): + if child_xpath in repeats or (encrypted and child_name == "media"): node_type = list if node_type == dict: @@ -225,7 +231,7 @@ def _flatten_dict(d, prefix): # hack: removing [1] index to be consistent across # surveys that have a single repitition of the # loop versus mutliple. - item_prefix[-1] += u"[%s]" % text(i + 1) + item_prefix[-1] += "[%s]" % text(i + 1) if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): @@ -256,13 +262,12 @@ def _flatten_dict_nest_repeats(d, prefix): if isinstance(item, dict): repeat = {} - for path, value in _flatten_dict_nest_repeats( - item, item_prefix): + for path, value in _flatten_dict_nest_repeats(item, item_prefix): # TODO: this only considers the first level of repeats - repeat.update({u"/".join(path[1:]): value}) + repeat.update({"/".join(path[1:]): value}) repeats.append(repeat) else: - repeats.append({u"/".join(item_prefix[1:]): item}) + repeats.append({"/".join(item_prefix[1:]): item}) yield (new_prefix, repeats) else: yield (new_prefix, value) @@ -300,7 +305,6 @@ def _get_all_attributes(node): class XFormInstanceParser(object): - def __init__(self, xml_str, data_dictionary): self.dd = data_dictionary self.parse(xml_str) @@ -308,18 +312,19 @@ def __init__(self, xml_str, data_dictionary): def parse(self, xml_str): self._xml_obj = clean_and_parse_xml(xml_str) self._root_node = self._xml_obj.documentElement - repeats = [e.get_abbreviated_xpath() - for e in self.dd.get_survey_elements_of_type(u"repeat")] + repeats = [ + e.get_abbreviated_xpath() + for e in self.dd.get_survey_elements_of_type("repeat") + ] - self._dict = _xml_node_to_dict(self._root_node, repeats, - self.dd.encrypted) + self._dict = _xml_node_to_dict(self._root_node, repeats, self.dd.encrypted) self._flat_dict = {} if self._dict is None: raise InstanceEmptyError for path, value in _flatten_dict_nest_repeats(self._dict, []): - self._flat_dict[u"/".join(path[1:])] = value + self._flat_dict["/".join(path[1:])] = value self._set_attributes() def get_root_node(self): @@ -348,17 +353,18 @@ def _set_attributes(self): # multiple xml tags, overriding and log when this occurs if key in self._attributes: logger = logging.getLogger("console_logger") - logger.debug("Skipping duplicate attribute: %s" - " with value %s" % (key, value)) + logger.debug( + "Skipping duplicate attribute: %s" " with value %s" % (key, value) + ) logger.debug(text(all_attributes)) else: self._attributes[key] = value def get_xform_id_string(self): - return self._attributes[u"id"] + return self._attributes["id"] def get_version(self): - return self._attributes.get(u"version") + return self._attributes.get("version") def get_flat_dict_with_attributes(self): result = self.to_flat_dict().copy() diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index 1e3176f56c..0947297ff5 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -4,8 +4,8 @@ """ import os import re -from future.moves.urllib.parse import urlparse -from future.moves.urllib.request import urlopen +from six.moves.urllib.parse import urlparse +from six.moves.urllib.request import urlopen import requests from django import forms @@ -28,37 +28,38 @@ from onadata.libs.utils.user_auth import get_user_default_project FORM_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('https://creativecommons.org/licenses/by/3.0/', - ugettext_lazy('Attribution CC BY')), - ('https://creativecommons.org/licenses/by-sa/3.0/', - ugettext_lazy('Attribution-ShareAlike CC BY-SA')), + ("No License", ugettext_lazy("No License")), + ( + "https://creativecommons.org/licenses/by/3.0/", + ugettext_lazy("Attribution CC BY"), + ), + ( + "https://creativecommons.org/licenses/by-sa/3.0/", + ugettext_lazy("Attribution-ShareAlike CC BY-SA"), + ), ) DATA_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('http://opendatacommons.org/licenses/pddl/summary/', - ugettext_lazy('PDDL')), - ('http://opendatacommons.org/licenses/by/summary/', - ugettext_lazy('ODC-BY')), - ('http://opendatacommons.org/licenses/odbl/summary/', - ugettext_lazy('ODBL')), + ("No License", ugettext_lazy("No License")), + ("http://opendatacommons.org/licenses/pddl/summary/", ugettext_lazy("PDDL")), + ("http://opendatacommons.org/licenses/by/summary/", ugettext_lazy("ODC-BY")), + ("http://opendatacommons.org/licenses/odbl/summary/", ugettext_lazy("ODBL")), ) PERM_CHOICES = ( - ('view', ugettext_lazy('Can view')), - ('edit', ugettext_lazy('Can edit')), - ('report', ugettext_lazy('Can submit to')), - ('remove', ugettext_lazy('Remove permissions')), + ("view", ugettext_lazy("Can view")), + ("edit", ugettext_lazy("Can edit")), + ("report", ugettext_lazy("Can submit to")), + ("remove", ugettext_lazy("Remove permissions")), ) VALID_XLSFORM_CONTENT_TYPES = [ - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'text/csv', - 'application/vnd.ms-excel' + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "text/csv", + "application/vnd.ms-excel", ] -VALID_FILE_EXTENSIONS = ['.xlsx', '.csv'] +VALID_FILE_EXTENSIONS = [".xlsx", ".csv"] def get_filename(response): @@ -70,11 +71,11 @@ def get_filename(response): # following format: # 'attachment; filename="ActApp_Survey_System.xlsx"; filename*=UTF-8\'\'ActApp_Survey_System.xlsx' # noqa cleaned_xls_file = "" - content = response.headers.get('content-disposition').split('; ') - counter = [a for a in content if a.startswith('filename=')] + content = response.headers.get("content-disposition").split("; ") + counter = [a for a in content if a.startswith("filename=")] if len(counter) >= 1: filename_key_val = counter[0] - filename = filename_key_val.split('=')[1].replace("\"", "") + filename = filename_key_val.split("=")[1].replace('"', "") name, extension = os.path.splitext(filename) if extension in VALID_FILE_EXTENSIONS and name: @@ -84,36 +85,40 @@ def get_filename(response): class DataLicenseForm(forms.Form): - """" + """ " Data license form. """ - value = forms.ChoiceField(choices=DATA_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'data-license'})) + + value = forms.ChoiceField( + choices=DATA_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "data-license"}), + ) class FormLicenseForm(forms.Form): """ Form license form. """ - value = forms.ChoiceField(choices=FORM_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'form-license'})) + + value = forms.ChoiceField( + choices=FORM_LICENSES_CHOICES, + widget=forms.Select(attrs={"disabled": "disabled", "id": "form-license"}), + ) class PermissionForm(forms.Form): """ Permission assignment form. """ + for_user = forms.CharField( widget=forms.TextInput( attrs={ - 'id': 'autocomplete', - 'data-provide': 'typeahead', - 'autocomplete': 'off' - }) + "id": "autocomplete", + "data-provide": "typeahead", + "autocomplete": "off", + } + ) ) perm_type = forms.ChoiceField(choices=PERM_CHOICES, widget=forms.Select()) @@ -129,14 +134,15 @@ class UserProfileForm(ModelForm): class Meta: model = UserProfile - exclude = ('user', 'created_by', 'num_of_submissions') + exclude = ("user", "created_by", "num_of_submissions") + email = forms.EmailField(widget=forms.TextInput()) def clean_metadata(self): """ Returns an empty dict if metadata is None. """ - metadata = self.cleaned_data.get('metadata') + metadata = self.cleaned_data.get("metadata") return metadata if metadata is not None else dict() @@ -145,43 +151,49 @@ class UserProfileFormRegister(forms.Form): """ User profile registration form. """ - first_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - last_name = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - city = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - country = forms.ChoiceField(widget=forms.Select(), required=False, - choices=COUNTRIES, initial='ZZ') - organization = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - home_page = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - twitter = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) + + first_name = forms.CharField( + widget=forms.TextInput(), required=True, max_length=255 + ) + last_name = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + city = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) + country = forms.ChoiceField( + widget=forms.Select(), required=False, choices=COUNTRIES, initial="ZZ" + ) + organization = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + home_page = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + twitter = forms.CharField(widget=forms.TextInput(), required=False, max_length=255) def save_user_profile(self, new_user): """ Creates and returns a new_user profile. """ - new_profile = \ - UserProfile(user=new_user, name=self.cleaned_data['first_name'], - city=self.cleaned_data['city'], - country=self.cleaned_data['country'], - organization=self.cleaned_data['organization'], - home_page=self.cleaned_data['home_page'], - twitter=self.cleaned_data['twitter']) + new_profile = UserProfile( + user=new_user, + name=self.cleaned_data["first_name"], + city=self.cleaned_data["city"], + country=self.cleaned_data["country"], + organization=self.cleaned_data["organization"], + home_page=self.cleaned_data["home_page"], + twitter=self.cleaned_data["twitter"], + ) new_profile.save() return new_profile # pylint: disable=too-many-ancestors # order of inheritance control order of form display -class RegistrationFormUserProfile(RegistrationFormUniqueEmail, - UserProfileFormRegister): +class RegistrationFormUserProfile(RegistrationFormUniqueEmail, UserProfileFormRegister): """ User profile registration form. """ + RESERVED_USERNAMES = settings.RESERVED_USERNAMES username = forms.CharField(widget=forms.TextInput(), max_length=30) email = forms.EmailField(widget=forms.TextInput()) @@ -191,133 +203,149 @@ def clean_username(self): """ Validate a new user username. """ - username = self.cleaned_data['username'].lower() + username = self.cleaned_data["username"].lower() if username in self.RESERVED_USERNAMES: raise forms.ValidationError( - _(u'%s is a reserved name, please choose another') % username) + _("%s is a reserved name, please choose another") % username + ) elif not self.legal_usernames_re.search(username): raise forms.ValidationError( - _(u'username may only contain alpha-numeric characters and ' - u'underscores')) + _( + "username may only contain alpha-numeric characters and " + "underscores" + ) + ) try: User.objects.get(username=username) except User.DoesNotExist: return username - raise forms.ValidationError(_(u'%s already exists') % username) + raise forms.ValidationError(_("%s already exists") % username) class SourceForm(forms.Form): """ Source document form. """ - source = forms.FileField(label=ugettext_lazy(u"Source document"), - required=True) + + source = forms.FileField(label=ugettext_lazy("Source document"), required=True) class SupportDocForm(forms.Form): """ Supporting document. """ - doc = forms.FileField(label=ugettext_lazy(u"Supporting document"), - required=True) + + doc = forms.FileField(label=ugettext_lazy("Supporting document"), required=True) class MediaForm(forms.Form): """ Media file upload form. """ - media = forms.FileField(label=ugettext_lazy(u"Media upload"), - required=True) + + media = forms.FileField(label=ugettext_lazy("Media upload"), required=True) def clean_media(self): """ Validate media upload file. """ - data_type = self.cleaned_data['media'].content_type - if data_type not in ['image/jpeg', 'image/png', 'audio/mpeg']: - raise forms.ValidationError('Only these media types are \ - allowed .png .jpg .mp3 .3gp .wav') + data_type = self.cleaned_data["media"].content_type + if data_type not in ["image/jpeg", "image/png", "audio/mpeg"]: + raise forms.ValidationError( + "Only these media types are \ + allowed .png .jpg .mp3 .3gp .wav" + ) class MapboxLayerForm(forms.Form): """ Mapbox layers form. """ - map_name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - attribution = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - link = forms.URLField(label=ugettext_lazy(u'JSONP url'), - required=True) + + map_name = forms.CharField(widget=forms.TextInput(), required=True, max_length=255) + attribution = forms.CharField( + widget=forms.TextInput(), required=False, max_length=255 + ) + link = forms.URLField(label=ugettext_lazy("JSONP url"), required=True) class QuickConverterFile(forms.Form): """ Uploads XLSForm form. """ - xls_file = forms.FileField( - label=ugettext_lazy(u'XLS File'), required=False) + + xls_file = forms.FileField(label=ugettext_lazy("XLS File"), required=False) class QuickConverterURL(forms.Form): """ Uploads XLSForm from a URL. """ - xls_url = forms.URLField(label=ugettext_lazy('XLS URL'), - required=False) + + xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) class QuickConverterDropboxURL(forms.Form): """ Uploads XLSForm from Dropbox. """ - dropbox_xls_url = forms.URLField( - label=ugettext_lazy('XLS URL'), required=False) + + dropbox_xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) class QuickConverterCsvFile(forms.Form): """ Uploads CSV XLSForm. """ - csv_url = forms.URLField( - label=ugettext_lazy('CSV URL'), required=False) + + csv_url = forms.URLField(label=ugettext_lazy("CSV URL"), required=False) class QuickConverterTextXlsForm(forms.Form): """ Uploads Text XLSForm. """ + text_xls_form = forms.CharField( - label=ugettext_lazy('XLSForm Representation'), required=False) + label=ugettext_lazy("XLSForm Representation"), required=False + ) class QuickConverterXmlFile(forms.Form): """ Uploads an XForm XML. """ - xml_file = forms.FileField( - label=ugettext_lazy(u'XML File'), required=False) + + xml_file = forms.FileField(label=ugettext_lazy("XML File"), required=False) class QuickConverterFloipFile(forms.Form): """ Uploads a FLOIP results data package descriptor file. """ + floip_file = forms.FileField( - label=ugettext_lazy(u'FlOIP results data packages descriptor File'), - required=False) + label=ugettext_lazy("FlOIP results data packages descriptor File"), + required=False, + ) # pylint: disable=too-many-ancestors -class QuickConverter(QuickConverterFile, QuickConverterURL, - QuickConverterDropboxURL, QuickConverterTextXlsForm, - QuickConverterCsvFile, QuickConverterXmlFile, - QuickConverterFloipFile): +class QuickConverter( + QuickConverterFile, + QuickConverterURL, + QuickConverterDropboxURL, + QuickConverterTextXlsForm, + QuickConverterCsvFile, + QuickConverterXmlFile, + QuickConverterFloipFile, +): """ Publish XLSForm and convert to XForm. """ + project = forms.IntegerField(required=False) validate = URLValidator() @@ -325,14 +353,13 @@ def clean_project(self): """ Project validation. """ - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is not None: try: # pylint: disable=attribute-defined-outside-init, no-member self._project = Project.objects.get(pk=int(project)) except (Project.DoesNotExist, ValueError): - raise forms.ValidationError( - _(u"Unknown project id: %s" % project)) + raise forms.ValidationError(_("Unknown project id: %s" % project)) return project @@ -345,95 +372,105 @@ def publish(self, user, id_string=None, created_by=None): # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None - if 'text_xls_form' in self.cleaned_data\ - and self.cleaned_data['text_xls_form'].strip(): - csv_data = self.cleaned_data['text_xls_form'] + if ( + "text_xls_form" in self.cleaned_data + and self.cleaned_data["text_xls_form"].strip() + ): + csv_data = self.cleaned_data["text_xls_form"] # assigning the filename to a random string (quick fix) import random - rand_name = "uploaded_form_%s.csv" % ''.join( - random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) - - cleaned_xls_file = \ - default_storage.save( - upload_to(None, rand_name, user.username), - ContentFile(csv_data.encode())) - if 'xls_file' in self.cleaned_data and\ - self.cleaned_data['xls_file']: - cleaned_xls_file = self.cleaned_data['xls_file'] - if 'floip_file' in self.cleaned_data and\ - self.cleaned_data['floip_file']: - cleaned_xls_file = self.cleaned_data['floip_file'] + + rand_name = "uploaded_form_%s.csv" % "".join( + random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) + ) + + cleaned_xls_file = default_storage.save( + upload_to(None, rand_name, user.username), + ContentFile(csv_data.encode()), + ) + if "xls_file" in self.cleaned_data and self.cleaned_data["xls_file"]: + cleaned_xls_file = self.cleaned_data["xls_file"] + if "floip_file" in self.cleaned_data and self.cleaned_data["floip_file"]: + cleaned_xls_file = self.cleaned_data["floip_file"] cleaned_url = ( - self.cleaned_data['xls_url'].strip() or - self.cleaned_data['dropbox_xls_url'] or - self.cleaned_data['csv_url']) + self.cleaned_data["xls_url"].strip() + or self.cleaned_data["dropbox_xls_url"] + or self.cleaned_data["csv_url"] + ) if cleaned_url: cleaned_xls_file = urlparse(cleaned_url) - cleaned_xls_file = \ - '_'.join(cleaned_xls_file.path.split('/')[-2:]) + cleaned_xls_file = "_".join(cleaned_xls_file.path.split("/")[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: response = requests.get(cleaned_url) - if response.headers.get('content-type') in \ - VALID_XLSFORM_CONTENT_TYPES and \ - response.status_code < 400: + if ( + response.headers.get("content-type") + in VALID_XLSFORM_CONTENT_TYPES + and response.status_code < 400 + ): cleaned_xls_file = get_filename(response) - cleaned_xls_file = \ - upload_to(None, cleaned_xls_file, user.username) + cleaned_xls_file = upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urlopen(cleaned_url).read()) - cleaned_xls_file = \ - default_storage.save(cleaned_xls_file, xls_data) + cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) - project = self.cleaned_data['project'] + project = self.cleaned_data["project"] if project is None: project = get_user_default_project(user) else: project = self._project - cleaned_xml_file = self.cleaned_data['xml_file'] + cleaned_xml_file = self.cleaned_data["xml_file"] if cleaned_xml_file: - return publish_xml_form(cleaned_xml_file, user, project, - id_string, created_by or user) + return publish_xml_form( + cleaned_xml_file, user, project, id_string, created_by or user + ) if cleaned_xls_file is None: raise forms.ValidationError( - _(u"XLSForm not provided, expecting either of these" - " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," - " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) + _( + "XLSForm not provided, expecting either of these" + " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," + " 'dropbox_xls_url', 'text_xls_form', 'floip_file'" + ) + ) # publish the xls - return publish_xls_form(cleaned_xls_file, user, project, - id_string, created_by or user) + return publish_xls_form( + cleaned_xls_file, user, project, id_string, created_by or user + ) class ActivateSMSSupportForm(forms.Form): """ Enable SMS support form. """ - enable_sms_support = forms.TypedChoiceField(coerce=lambda x: x == 'True', - choices=((False, 'No'), - (True, 'Yes')), - widget=forms.Select, - label=ugettext_lazy( - u"Enable SMS Support")) - sms_id_string = forms.CharField(max_length=50, required=True, - label=ugettext_lazy(u"SMS Keyword")) + + enable_sms_support = forms.TypedChoiceField( + coerce=lambda x: x == "True", + choices=((False, "No"), (True, "Yes")), + widget=forms.Select, + label=ugettext_lazy("Enable SMS Support"), + ) + sms_id_string = forms.CharField( + max_length=50, required=True, label=ugettext_lazy("SMS Keyword") + ) def clean_sms_id_string(self): """ SMS id_string validation. """ - sms_id_string = self.cleaned_data.get('sms_id_string', '').strip() + sms_id_string = self.cleaned_data.get("sms_id_string", "").strip() - if not re.match(r'^[a-z0-9\_\-]+$', sms_id_string): - raise forms.ValidationError(u"id_string can only contain alphanum" - u" characters") + if not re.match(r"^[a-z0-9\_\-]+$", sms_id_string): + raise forms.ValidationError( + "id_string can only contain alphanum" " characters" + ) return sms_id_string @@ -442,8 +479,9 @@ class ExternalExportForm(forms.Form): """ XLS reports form. """ - template_name = forms.CharField(label='Template Name', max_length=20) - template_token = forms.URLField(label='Template URL', max_length=100) + + template_name = forms.CharField(label="Template Name", max_length=20) + template_token = forms.URLField(label="Template URL", max_length=100) # Deprecated diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 691a2d1a8c..117d118b0f 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -9,17 +9,20 @@ from django.db import models from django.db.models.signals import post_save, pre_save from django.utils.translation import ugettext_lazy -from django.utils.encoding import python_2_unicode_compatible from guardian.shortcuts import get_perms_for_model, assign_perm from guardian.models import UserObjectPermissionBase from guardian.models import GroupObjectPermissionBase from rest_framework.authtoken.models import Token +from six import python_2_unicode_compatible from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists from onadata.apps.main.signals import ( - set_api_permissions, send_inactive_user_email, send_activation_email) + set_api_permissions, + send_inactive_user_email, + send_activation_email, +) -REQUIRE_AUTHENTICATION = 'REQUIRE_ODK_AUTHENTICATION' +REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" @python_2_unicode_compatible @@ -29,8 +32,7 @@ class UserProfile(models.Model): """ # This field is required. - user = models.OneToOneField( - User, related_name='profile', on_delete=models.CASCADE) + user = models.OneToOneField(User, related_name="profile", on_delete=models.CASCADE) # Other fields here name = models.CharField(max_length=255, blank=True) @@ -41,18 +43,19 @@ class UserProfile(models.Model): twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( - default=False, - verbose_name=ugettext_lazy("Require Phone Authentication")) + default=False, verbose_name=ugettext_lazy("Require Phone Authentication") + ) address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) created_by = models.ForeignKey( - User, null=True, blank=True, on_delete=models.SET_NULL) + User, null=True, blank=True, on_delete=models.SET_NULL + ) num_of_submissions = models.IntegerField(default=0) metadata = JSONField(default=dict, blank=True) date_modified = models.DateTimeField(auto_now=True) def __str__(self): - return u'%s[%s]' % (self.name, self.user.username) + return "%s[%s]" % (self.name, self.user.username) @property def gravatar(self): @@ -68,22 +71,22 @@ def twitter_clean(self): return self.twitter[1:] return self.twitter - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): + def save( + self, force_insert=False, force_update=False, using=None, update_fields=None + ): # Override default save method to set settings configured require_auth # value if self.pk is None and hasattr(settings, REQUIRE_AUTHENTICATION): self.require_auth = getattr(settings, REQUIRE_AUTHENTICATION) - super(UserProfile, self).save(force_insert, force_update, using, - update_fields) + super(UserProfile, self).save(force_insert, force_update, using, update_fields) class Meta: - app_label = 'main' + app_label = "main" permissions = ( - ('can_add_project', "Can add a project to an organization"), - ('can_add_xform', "Can add/upload an xform to user profile"), - ('view_profile', "Can view user profile"), + ("can_add_project", "Can add a project to an organization"), + ("can_add_xform", "Can add/upload an xform to user profile"), + ("view_profile", "Can view user profile"), ) @@ -101,47 +104,46 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) -def set_kpi_formbuilder_permissions( - sender, instance=None, created=False, **kwargs): +def set_kpi_formbuilder_permissions(sender, instance=None, created=False, **kwargs): if created: - kpi_formbuilder_url = hasattr(settings, 'KPI_FORMBUILDER_URL') and\ - settings.KPI_FORMBUILDER_URL + kpi_formbuilder_url = ( + hasattr(settings, "KPI_FORMBUILDER_URL") and settings.KPI_FORMBUILDER_URL + ) if kpi_formbuilder_url: requests.post( - "%s/%s" % ( - kpi_formbuilder_url, - 'grant-default-model-level-perms' - ), - headers={ - 'Authorization': 'Token %s' % instance.user.auth_token - } + "%s/%s" % (kpi_formbuilder_url, "grant-default-model-level-perms"), + headers={"Authorization": "Token %s" % instance.user.auth_token}, ) -post_save.connect(create_auth_token, sender=User, dispatch_uid='auth_token') +post_save.connect(create_auth_token, sender=User, dispatch_uid="auth_token") post_save.connect( - send_inactive_user_email, sender=User, - dispatch_uid='send_inactive_user_email') + send_inactive_user_email, sender=User, dispatch_uid="send_inactive_user_email" +) pre_save.connect( - send_activation_email, sender=User, - dispatch_uid='send_activation_email' + send_activation_email, sender=User, dispatch_uid="send_activation_email" ) -post_save.connect(set_api_permissions, sender=User, - dispatch_uid='set_api_permissions') +post_save.connect(set_api_permissions, sender=User, dispatch_uid="set_api_permissions") -post_save.connect(set_object_permissions, sender=UserProfile, - dispatch_uid='set_object_permissions') +post_save.connect( + set_object_permissions, sender=UserProfile, dispatch_uid="set_object_permissions" +) -post_save.connect(set_kpi_formbuilder_permissions, sender=UserProfile, - dispatch_uid='set_kpi_formbuilder_permission') +post_save.connect( + set_kpi_formbuilder_permissions, + sender=UserProfile, + dispatch_uid="set_kpi_formbuilder_permission", +) class UserProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) class UserProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" + content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index abc963ef1b..332964273b 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -6,8 +6,8 @@ import re import socket from builtins import open -from future.moves.urllib.error import URLError -from future.moves.urllib.request import urlopen +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen from io import StringIO from tempfile import NamedTemporaryFile @@ -29,27 +29,31 @@ from onadata.apps.logger.views import submission from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models import DataDictionary -from onadata.libs.utils.common_tools import (filename_from_disposition, - get_response_content) +from onadata.libs.utils.common_tools import ( + filename_from_disposition, + get_response_content, +) from onadata.libs.utils.user_auth import get_user_default_project class TestBase(PyxformMarkdown, TransactionTestCase): maxDiff = None - surveys = ['transport_2011-07-25_19-05-49', - 'transport_2011-07-25_19-05-36', - 'transport_2011-07-25_19-06-01', - 'transport_2011-07-25_19-06-14'] + surveys = [ + "transport_2011-07-25_19-05-49", + "transport_2011-07-25_19-05-36", + "transport_2011-07-25_19-06-01", + "transport_2011-07-25_19-06-14", + ] this_directory = os.path.abspath(os.path.dirname(__file__)) def setUp(self): self.maxDiff = None self._create_user_and_login() - self.base_url = 'http://testserver' + self.base_url = "http://testserver" self.factory = RequestFactory() def _fixture_path(self, *args): - return os.path.join(os.path.dirname(__file__), 'fixtures', *args) + return os.path.join(os.path.dirname(__file__), "fixtures", *args) def _create_user(self, username, password, create_profile=False): user, created = User.objects.get_or_create(username=username) @@ -74,8 +78,7 @@ def _logout(self, client=None): client = self.client client.logout() - def _create_user_and_login(self, username="bob", password="bob", - factory=None): + def _create_user_and_login(self, username="bob", password="bob", factory=None): self.login_username = username self.login_password = password self.user = self._create_user(username, password, create_profile=True) @@ -85,34 +88,35 @@ def _create_user_and_login(self, username="bob", password="bob", self.anon = Client() def _publish_xls_file(self, path): - if not path.startswith('/%s/' % self.user.username): + if not path.startswith("/%s/" % self.user.username): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as f: + with open(path, "rb") as f: xls_file = InMemoryUploadedFile( f, - 'xls_file', + "xls_file", os.path.abspath(os.path.basename(path)), - 'application/vnd.ms-excel', + "application/vnd.ms-excel", os.path.getsize(path), - None) - if not hasattr(self, 'project'): + None, + ) + if not hasattr(self, "project"): self.project = get_user_default_project(self.user) DataDictionary.objects.create( - created_by=self.user, - user=self.user, - xls=xls_file, - project=self.project) + created_by=self.user, user=self.user, xls=xls_file, project=self.project + ) def _publish_xlsx_file(self): - path = os.path.join(self.this_directory, 'fixtures', 'exp.xlsx') + path = os.path.join(self.this_directory, "fixtures", "exp.xlsx") pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) # make sure publishing the survey worked self.assertEqual(XForm.objects.count(), pre_count + 1) def _publish_xlsx_file_with_external_choices(self): - path = os.path.join(self.this_directory, 'fixtures', 'external_choice_form_v1.xlsx') + path = os.path.join( + self.this_directory, "fixtures", "external_choice_form_v1.xlsx" + ) pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) # make sure publishing the survey worked @@ -122,41 +126,68 @@ def _publish_xls_file_and_set_xform(self, path): count = XForm.objects.count() self._publish_xls_file(path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] - def _share_form_data(self, id_string='transportation_2011_07_25'): + def _share_form_data(self, id_string="transportation_2011_07_25"): xform = XForm.objects.get(id_string=id_string) xform.shared_data = True xform.save() def _publish_transportation_form(self): xls_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() TestBase._publish_xls_file(self, xls_path) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] def _submit_transport_instance(self, survey_at=0): s = self.surveys[survey_at] - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + ) def _submit_transport_instance_w_uuid(self, name): - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances_w_uuid', name, name + '.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances_w_uuid", + name, + name + ".xml", + ) + ) def _submit_transport_instance_w_attachment(self, survey_at=0): s = self.surveys[survey_at] media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', s, media_file)) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ), + ) self.attachment = Attachment.objects.all().reverse()[0] self.attachment_media_file = self.attachment.media_file @@ -165,48 +196,54 @@ def _publish_transportation_form_and_submit_instance(self): self._submit_transport_instance() def _make_submissions_gps(self): - surveys = ['gps_1980-01-23_20-52-08', - 'gps_1980-01-23_21-21-33', ] + surveys = [ + "gps_1980-01-23_20-52-08", + "gps_1980-01-23_21-21-33", + ] for survey in surveys: - path = self._fixture_path('gps', 'instances', survey + '.xml') + path = self._fixture_path("gps", "instances", survey + ".xml") self._make_submission(path) - def _make_submission(self, path, username=None, add_uuid=False, - forced_submission_time=None, auth=None, client=None): + def _make_submission( + self, + path, + username=None, + add_uuid=False, + forced_submission_time=None, + auth=None, + client=None, + ): # store temporary file with dynamic uuid self.factory = APIRequestFactory() if auth is None: - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") tmp_file = None if add_uuid: - tmp_file = NamedTemporaryFile(delete=False, mode='w') + tmp_file = NamedTemporaryFile(delete=False, mode="w") split_xml = None - with open(path, encoding='utf-8') as _file: - split_xml = re.split(r'()', _file.read()) + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) - split_xml[1:1] = [ - '%s' % self.xform.uuid - ] - tmp_file.write(''.join(split_xml)) + split_xml[1:1] = ["%s" % self.xform.uuid] + tmp_file.write("".join(split_xml)) path = tmp_file.name tmp_file.close() - with open(path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} if username is None: username = self.user.username - url_prefix = '%s/' % username if username else '' - url = '/%ssubmission' % url_prefix + url_prefix = "%s/" % username if username else "" + url = "/%ssubmission" % url_prefix request = self.factory.post(url, post_data) - request.user = authenticate(username=auth.username, - password=auth.password) + request.user = authenticate(username=auth.username, password=auth.password) self.response = submission(request, username=username) @@ -215,7 +252,7 @@ def _make_submission(self, path, username=None, add_uuid=False, self.response = submission(request, username=username) if forced_submission_time: - instance = Instance.objects.order_by('-pk').all()[0] + instance = Instance.objects.order_by("-pk").all()[0] instance.date_created = forced_submission_time instance.json = instance.get_full_dict() instance.save() @@ -226,33 +263,27 @@ def _make_submission(self, path, username=None, add_uuid=False, os.unlink(tmp_file.name) def _make_submission_w_attachment(self, path, attachment_path): - with open(path, encoding='utf-8') as f: - data = {'xml_submission_file': f} + with open(path, encoding="utf-8") as f: + data = {"xml_submission_file": f} if attachment_path is not None: if isinstance(attachment_path, list): for c in range(len(attachment_path)): - data['media_file_{}'.format(c)] = open( - attachment_path[c], 'rb') + data["media_file_{}".format(c)] = open(attachment_path[c], "rb") else: - data['media_file'] = open( - attachment_path, 'rb') + data["media_file"] = open(attachment_path, "rb") - url = '/%s/submission' % self.user.username - auth = DigestAuth('bob', 'bob') + url = "/%s/submission" % self.user.username + auth = DigestAuth("bob", "bob") self.factory = APIRequestFactory() request = self.factory.post(url, data) - request.user = authenticate(username='bob', - password='bob') - self.response = submission(request, - username=self.user.username) + request.user = authenticate(username="bob", password="bob") + self.response = submission(request, username=self.user.username) if auth and self.response.status_code == 401: request.META.update(auth(request.META, self.response)) - self.response = submission(request, - username=self.user.username) + self.response = submission(request, username=self.user.username) - def _make_submissions(self, username=None, add_uuid=False, - should_store=True): + def _make_submissions(self, username=None, add_uuid=False, should_store=True): """Make test fixture submissions to current xform. :param username: submit under this username, default None. @@ -260,16 +291,23 @@ def _make_submissions(self, username=None, add_uuid=False, :param should_store: should submissions be save, default True. """ - paths = [os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', s, s + '.xml') for s in self.surveys] + paths = [ + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + for s in self.surveys + ] pre_count = Instance.objects.count() for path in paths: self._make_submission(path, username, add_uuid) - post_count = pre_count + len(self.surveys) if should_store\ - else pre_count + post_count = pre_count + len(self.surveys) if should_store else pre_count self.assertEqual(Instance.objects.count(), post_count) self.assertEqual(self.xform.instances.count(), post_count) xform = XForm.objects.get(pk=self.xform.pk) @@ -284,26 +322,25 @@ def _check_url(self, url, timeout=1): pass return False - def _internet_on(self, url='http://74.125.113.99'): + def _internet_on(self, url="http://74.125.113.99"): # default value is some google IP return self._check_url(url) def _set_auth_headers(self, username, password): return { - 'HTTP_AUTHORIZATION': - 'Basic ' + base64.b64encode(( - '%s:%s' % (username, password)).encode( - 'utf-8')).decode('utf-8'), + "HTTP_AUTHORIZATION": "Basic " + + base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode( + "utf-8" + ), } - def _get_authenticated_client( - self, url, username='bob', password='bob', extra={}): + def _get_authenticated_client(self, url, username="bob", password="bob", extra={}): client = DigestClient() # request with no credentials req = client.get(url, {}, **extra) self.assertEqual(req.status_code, 401) # apply credentials - client.set_authorization(username, password, 'Digest') + client.set_authorization(username, password, "Digest") return client def _set_mock_time(self, mock_time): @@ -318,24 +355,37 @@ def _set_require_auth(self, auth=True): def _get_digest_client(self): self._set_require_auth(True) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") return client def _publish_submit_geojson(self): path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "geolocation", "GeoLocationForm.xlsx") + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm.xlsx", + ) self._publish_xls_file_and_set_xform(path) - view = XFormViewSet.as_view({'post': 'csv_import'}) - csv_import = \ - open(os.path.join(settings.PROJECT_ROOT, 'apps', 'main', - 'tests', 'fixtures', 'geolocation', - 'GeoLocationForm_2015_01_15_01_28_45.csv'), - encoding='utf-8') - post_data = {'csv_file': csv_import} - request = self.factory.post('/', data=post_data, **self.extra) + view = XFormViewSet.as_view({"post": "csv_import"}) + csv_import = open( + os.path.join( + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "geolocation", + "GeoLocationForm_2015_01_15_01_28_45.csv", + ), + encoding="utf-8", + ) + post_data = {"csv_file": csv_import} + request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) @@ -343,30 +393,34 @@ def _publish_markdown(self, md_xlsform, user, project=None, **kwargs): """ Publishes a markdown XLSForm. """ - kwargs['name'] = 'data' + kwargs["name"] = "data" survey = self.md_to_pyxform_survey(md_xlsform, kwargs=kwargs) - survey['sms_keyword'] = survey['id_string'] - if not project or not hasattr(self, 'project'): + survey["sms_keyword"] = survey["id_string"] + if not project or not hasattr(self, "project"): project = get_user_default_project(user) - xform = DataDictionary(created_by=user, user=user, - xml=survey.to_xml(), json=survey.to_json(), - project=project) + xform = DataDictionary( + created_by=user, + user=user, + xml=survey.to_xml(), + json=survey.to_json(), + project=project, + ) xform.save() return xform def _test_csv_response(self, response, csv_file_path): headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) __, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") data = get_response_content(response) reader = csv.DictReader(StringIO(data)) data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: @@ -376,7 +430,7 @@ def _test_csv_response(self, response, csv_file_path): def _test_csv_files(self, csv_file, csv_file_path): reader = csv.DictReader(csv_file) data = [_ for _ in reader] - with open(csv_file_path, encoding='utf-8') as test_file: + with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): if None in row: diff --git a/onadata/apps/main/tests/test_form_enter_data.py b/onadata/apps/main/tests/test_form_enter_data.py index 27b1ebca23..0eb2b18186 100644 --- a/onadata/apps/main/tests/test_form_enter_data.py +++ b/onadata/apps/main/tests/test_form_enter_data.py @@ -7,7 +7,7 @@ from django.core.validators import URLValidator from django.test import RequestFactory from django.urls import reverse -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from httmock import HTTMock, urlmatch from nose import SkipTest diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 6cd3497550..9bfcafc3c2 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -16,7 +16,7 @@ from django.core.files.uploadedfile import UploadedFile from django.urls import reverse from django_digest.test import Client as DigestClient -from future.utils import iteritems +from six import iteritems from mock import patch from onadata.apps.logger.models import XForm @@ -28,24 +28,23 @@ from onadata.libs.utils.common_tags import MONGO_STRFTIME from onadata.libs.utils.common_tools import get_response_content -uuid_regex = re.compile( - r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) +uuid_regex = re.compile(r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) class TestProcess(TestBase): - loop_str = 'loop_over_transport_types_frequency' - frequency_str = 'frequency_to_referral_facility' - ambulance_key = '%s/ambulance/%s' % (loop_str, frequency_str) - bicycle_key = '%s/bicycle/%s' % (loop_str, frequency_str) - other_key = '%s/other/%s' % (loop_str, frequency_str) - taxi_key = '%s/taxi/%s' % (loop_str, frequency_str) - transport_ambulance_key = u'transport/%s' % ambulance_key - transport_bicycle_key = u'transport/%s' % bicycle_key + loop_str = "loop_over_transport_types_frequency" + frequency_str = "frequency_to_referral_facility" + ambulance_key = "%s/ambulance/%s" % (loop_str, frequency_str) + bicycle_key = "%s/bicycle/%s" % (loop_str, frequency_str) + other_key = "%s/other/%s" % (loop_str, frequency_str) + taxi_key = "%s/taxi/%s" % (loop_str, frequency_str) + transport_ambulance_key = "transport/%s" % ambulance_key + transport_bicycle_key = "transport/%s" % bicycle_key uuid_to_submission_times = { - '5b2cc313-fc09-437e-8149-fcd32f695d41': '2013-02-14T15:37:21', - 'f3d8dc65-91a6-4d0f-9e97-802128083390': '2013-02-14T15:37:22', - '9c6f3468-cfda-46e8-84c1-75458e72805d': '2013-02-14T15:37:23', - '9f0a1508-c3b7-4c99-be00-9b237c26bcbf': '2013-02-14T15:37:24' + "5b2cc313-fc09-437e-8149-fcd32f695d41": "2013-02-14T15:37:21", + "f3d8dc65-91a6-4d0f-9e97-802128083390": "2013-02-14T15:37:22", + "9c6f3468-cfda-46e8-84c1-75458e72805d": "2013-02-14T15:37:23", + "9f0a1508-c3b7-4c99-be00-9b237c26bcbf": "2013-02-14T15:37:24", } def setUp(self): @@ -67,54 +66,71 @@ def _update_dynamic_data(self): """ Update stuff like submission time so we can compare within out fixtures """ - for (uuid, submission_time) in iteritems( - self.uuid_to_submission_times): + for (uuid, submission_time) in iteritems(self.uuid_to_submission_times): i = self.xform.instances.get(uuid=uuid) - i.date_created = pytz.timezone('UTC').localize( - datetime.strptime(submission_time, MONGO_STRFTIME)) + i.date_created = pytz.timezone("UTC").localize( + datetime.strptime(submission_time, MONGO_STRFTIME) + ) i.json = i.get_full_dict() i.save() def test_uuid_submit(self): self._publish_xls_file() - survey = 'transport_2011-07-25_19-05-49' + survey = "transport_2011-07-25_19-05-49" path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml') + self.this_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) with open(path) as f: - post_data = {'xml_submission_file': f, 'uuid': self.xform.uuid} - url = '/submission' + post_data = {"xml_submission_file": f, "uuid": self.xform.uuid} + url = "/submission" self.response = self.client.post(url, post_data) def test_publish_xlsx_file(self): self._publish_xlsx_file() - @patch('onadata.apps.main.forms.requests') - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.requests") + @patch("onadata.apps.main.forms.urlopen") def test_google_url_upload(self, mock_urlopen, mock_requests): if self._internet_on(url="http://google.com"): - xls_url = "https://docs.google.com/spreadsheet/pub?"\ + xls_url = ( + "https://docs.google.com/spreadsheet/pub?" "key=0AvhZpT7ZLAWmdDhISGhqSjBOSl9XdXd5SHZHUUE2RFE&output=xlsx" + ) pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xlsx") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xlsx", + ) + + xls_file = open(path, "rb") mock_response = requests.Response() mock_response.status_code = 200 mock_response.headers = { - 'content-type': ("application/vnd.openxmlformats-" - "officedocument.spreadsheetml.sheet"), - 'content-disposition': ( + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "content-disposition": ( 'attachment; filename="transportation.' - 'xlsx"; filename*=UTF-8\'\'transportation.xlsx') + "xlsx\"; filename*=UTF-8''transportation.xlsx" + ), } mock_requests.get.return_value = mock_response mock_urlopen.return_value = xls_file - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) mock_urlopen.assert_called_with(xls_url) mock_requests.get.assert_called_with(xls_url) @@ -124,21 +140,28 @@ def test_google_url_upload(self, mock_urlopen, mock_requests): self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch('onadata.apps.main.forms.urlopen') + @patch("onadata.apps.main.forms.urlopen") def test_url_upload(self, mock_urlopen): if self._internet_on(url="http://google.com"): - xls_url = 'https://ona.io/examples/forms/tutorial/form.xlsx' + xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() path = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xlsx") - - xls_file = open(path, 'rb') + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "transportation.xlsx", + ) + + xls_file = open(path, "rb") mock_urlopen.return_value = xls_file - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) mock_urlopen.assert_called_with(xls_url) # cleanup the resources @@ -149,10 +172,9 @@ def test_url_upload(self, mock_urlopen): self.assertEqual(XForm.objects.count(), pre_count + 1) def test_bad_url_upload(self): - xls_url = 'formhuborg/pld/forms/transportation_2011_07_25/form.xlsx' + xls_url = "formhuborg/pld/forms/transportation_2011_07_25/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post("/%s/" % self.user.username, {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count) @@ -167,35 +189,36 @@ def test_upload_all_xls(self): success = True for root, sub_folders, filenames in os.walk(root_dir): # ignore files that don't end in '.xlsx' - for filename in fnmatch.filter(filenames, '*.xlsx'): - success = self._publish_file(os.path.join(root, filename), - False) + for filename in fnmatch.filter(filenames, "*.xlsx"): + success = self._publish_file(os.path.join(root, filename), False) if success: # delete it so we don't have id_string conflicts if self.xform: self.xform.delete() self.xform = None - print('finished sub-folder %s' % root) + print("finished sub-folder %s" % root) self.assertEqual(success, True) def test_url_upload_non_dot_xls_path(self): if self._internet_on(): - xls_url = 'http://formhub.org/formhub_u/forms/tutorial/form.xlsx' + xls_url = "http://formhub.org/formhub_u/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post('/%s/' % self.user.username, - {'xls_url': xls_url}) + response = self.client.post( + "/%s/" % self.user.username, {"xls_url": xls_url} + ) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) def test_not_logged_in_cannot_upload(self): - path = os.path.join(self.this_directory, "fixtures", "transportation", - "transportation.xlsx") - if not path.startswith('/%s/' % self.user.username): + path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) + if not path.startswith("/%s/" % self.user.username): path = os.path.join(self.this_directory, path) - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - return self.client.post('/%s/' % self.user.username, post_data) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + return self.client.post("/%s/" % self.user.username, post_data) def _publish_file(self, xls_path, strict=True): """ @@ -205,7 +228,7 @@ def _publish_file(self, xls_path, strict=True): TestBase._publish_xls_file(self, xls_path) # make sure publishing the survey worked if XForm.objects.count() != pre_count + 1: - print('\nPublish Failure for file: %s' % xls_path) + print("\nPublish Failure for file: %s" % xls_path) if strict: self.assertEqual(XForm.objects.count(), pre_count + 1) else: @@ -214,58 +237,62 @@ def _publish_file(self, xls_path, strict=True): return True def _publish_xls_file(self): - xls_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + xls_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) self._publish_file(xls_path) self.assertEqual(self.xform.id_string, "transportation_2011_07_25") def _check_formlist(self): - url = '/%s/formList' % self.user.username + url = "/%s/formList" % self.user.username client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(url) - self.download_url = \ - 'http://testserver/%s/forms/%s/form.xml'\ - % (self.user.username, self.xform.pk) - md5_hash = md5(self.xform.xml.encode('utf-8')).hexdigest() + self.download_url = "http://testserver/%s/forms/%s/form.xml" % ( + self.user.username, + self.xform.pk, + ) + md5_hash = md5(self.xform.xml.encode("utf-8")).hexdigest() expected_content = """ transportation_2011_07_25transportation_2011_07_252014111md5:%(hash)s%(download_url)s""" # noqa expected_content = expected_content % { - 'download_url': self.download_url, - 'hash': md5_hash + "download_url": self.download_url, + "hash": md5_hash, } - self.assertEqual(response.content.decode('utf-8'), expected_content) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue(response.has_header('Date')) + self.assertEqual(response.content.decode("utf-8"), expected_content) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("Date")) def _download_xform(self): client = DigestClient() - client.set_authorization('bob', 'bob') + client.set_authorization("bob", "bob") response = client.get(self.download_url) response_doc = minidom.parseString(response.content) - xml_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation.xml") - with open(xml_path, 'rb') as xml_file: + xml_path = os.path.join( + self.this_directory, "fixtures", "transportation", "transportation.xml" + ) + with open(xml_path, "rb") as xml_file: expected_doc = minidom.parse(xml_file) model_node = [ - n for n in - response_doc.getElementsByTagName("h:head")[0].childNodes - if n.nodeType == Node.ELEMENT_NODE and - n.tagName == "model"][0] + n + for n in response_doc.getElementsByTagName("h:head")[0].childNodes + if n.nodeType == Node.ELEMENT_NODE and n.tagName == "model" + ][0] # check for UUID and remove - uuid_nodes = [node for node in model_node.childNodes - if node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + uuid_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(uuid_nodes), 1) uuid_node = uuid_nodes[0] uuid_node.setAttribute("calculate", "''") - response_xml = response_doc.toxml().replace( - self.xform.version, u"201411120717") + response_xml = response_doc.toxml().replace(self.xform.version, "201411120717") # check content without UUID self.assertEqual(response_xml, expected_doc.toxml()) @@ -281,90 +308,98 @@ def _check_data_dictionary(self): qs = DataDictionary.objects.filter(user=self.user) self.assertEqual(qs.count(), 1) self.data_dictionary = DataDictionary.objects.all()[0] - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers.json" + ) + ) as f: expected_list = json.load(f) self.assertEqual(self.data_dictionary.get_headers(), expected_list) # test to make sure the headers in the actual csv are as expected actual_csv = self._get_csv_() - with open(os.path.join(self.this_directory, "fixtures", - "transportation", "headers_csv.json")) as f: + with open( + os.path.join( + self.this_directory, "fixtures", "transportation", "headers_csv.json" + ) + ) as f: expected_list = json.load(f) self.assertEqual(sorted(next(actual_csv)), sorted(expected_list)) def _check_data_for_csv_export(self): data = [ - {"available_transportation_types_to_referral_facility/ambulance": - True, - "available_transportation_types_to_referral_facility/bicycle": - True, - self.ambulance_key: "daily", - self.bicycle_key: "weekly" - }, + { + "available_transportation_types_to_referral_facility/ambulance": True, + "available_transportation_types_to_referral_facility/bicycle": True, + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + }, {}, - {"available_transportation_types_to_referral_facility/ambulance": - True, - self.ambulance_key: "weekly", - }, - {"available_transportation_types_to_referral_facility/taxi": True, - "available_transportation_types_to_referral_facility/other": True, - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - self.other_key: "other", - } + { + "available_transportation_types_to_referral_facility/ambulance": True, + self.ambulance_key: "weekly", + }, + { + "available_transportation_types_to_referral_facility/taxi": True, + "available_transportation_types_to_referral_facility/other": True, + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + self.other_key: "other", + }, ] for d_from_db in self.data_dictionary.get_data_for_excel(): test_dict = {} for (k, v) in iteritems(d_from_db): - if (k not in [u'_xform_id_string', u'meta/instanceID', - '_version', '_id', 'image1']) and v: - new_key = k[len('transport/'):] + if ( + k + not in [ + "_xform_id_string", + "meta/instanceID", + "_version", + "_id", + "image1", + ] + ) and v: + new_key = k[len("transport/") :] test_dict[new_key] = d_from_db[k] self.assertTrue(test_dict in data, (test_dict, data)) data.remove(test_dict) self.assertEquals(data, []) def _check_group_xpaths_do_not_appear_in_dicts_for_export(self): - uuid = u'uuid:f3d8dc65-91a6-4d0f-9e97-802128083390' - instance = self.xform.instances.get(uuid=uuid.split(':')[1]) + uuid = "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390" + instance = self.xform.instances.get(uuid=uuid.split(":")[1]) expected_dict = { - u"transportation": { - u"meta": { - u"instanceID": uuid - }, - u"transport": { - u"loop_over_transport_types_frequency": {u"bicycle": { - u"frequency_to_referral_facility": u"weekly" - }, - u"ambulance": { - u"frequency_to_referral_facility": u"daily" - } + "transportation": { + "meta": {"instanceID": uuid}, + "transport": { + "loop_over_transport_types_frequency": { + "bicycle": {"frequency_to_referral_facility": "weekly"}, + "ambulance": {"frequency_to_referral_facility": "daily"}, }, - u"available_transportation_types_to_referral_facility": - u"ambulance bicycle", - } + "available_transportation_types_to_referral_facility": "ambulance bicycle", + }, } } self.assertEqual(instance.get_dict(flat=False), expected_dict) expected_dict = { - u"transport/available_transportation_types_to_referral_facility": - u"ambulance bicycle", - self.transport_ambulance_key: u"daily", - self.transport_bicycle_key: u"weekly", - u"_xform_id_string": u"transportation_2011_07_25", - u"_version": u"2014111", - u"meta/instanceID": uuid + "transport/available_transportation_types_to_referral_facility": "ambulance bicycle", + self.transport_ambulance_key: "daily", + self.transport_bicycle_key: "weekly", + "_xform_id_string": "transportation_2011_07_25", + "_version": "2014111", + "meta/instanceID": uuid, } self.assertEqual(instance.get_dict(), expected_dict) def _get_csv_(self): # todo: get the csv.reader to handle unicode as done here: # http://docs.python.org/library/csv.html#examples - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -372,18 +407,22 @@ def _get_csv_(self): return csv.reader(actual_lines) def _check_csv_export_first_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) test_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.csv") + self.this_directory, "fixtures", "transportation", "transportation.csv" + ) self._test_csv_response(response, test_file_path) def _check_csv_export_second_pass(self): - url = reverse('csv_export', kwargs={ - 'username': self.user.username, 'id_string': self.xform.id_string}) + url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) actual_csv = get_response_content(response) @@ -391,55 +430,72 @@ def _check_csv_export_second_pass(self): actual_csv = csv.reader(actual_lines) headers = next(actual_csv) data = [ - {"image1": "1335783522563.jpg", - 'meta/instanceID': 'uuid:5b2cc313-fc09-437e-8149-fcd32f695d41', - '_uuid': '5b2cc313-fc09-437e-8149-fcd32f695d41', - '_submission_time': '2013-02-14T15:37:21', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '1', '_media_count': '0', - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - "available_transportation_types_to_referral_facility/bicycle": - "True", - self.ambulance_key: "daily", - self.bicycle_key: "weekly", - "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", - '_uuid': 'f3d8dc65-91a6-4d0f-9e97-802128083390', - '_submission_time': '2013-02-14T15:37:22', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/ambulance": - "True", - self.ambulance_key: "weekly", - "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", - '_uuid': '9c6f3468-cfda-46e8-84c1-75458e72805d', - '_submission_time': '2013-02-14T15:37:23', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - }, - {"available_transportation_types_to_referral_facility/taxi": - "True", - "available_transportation_types_to_referral_facility/other": - "True", - "available_transportation_types_to_referral_facility_other": - "camel", - self.taxi_key: "daily", - "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", - '_uuid': '9f0a1508-c3b7-4c99-be00-9b237c26bcbf', - '_submission_time': '2013-02-14T15:37:24', - '_tags': '', '_notes': '', '_version': '2014111', '_duration': '', - '_submitted_by': 'bob', '_total_media': '0', '_media_count': '0', - '_media_all_received': 'True' - } + { + "image1": "1335783522563.jpg", + "meta/instanceID": "uuid:5b2cc313-fc09-437e-8149-fcd32f695d41", + "_uuid": "5b2cc313-fc09-437e-8149-fcd32f695d41", + "_submission_time": "2013-02-14T15:37:21", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "1", + "_media_count": "0", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + "available_transportation_types_to_referral_facility/bicycle": "True", + self.ambulance_key: "daily", + self.bicycle_key: "weekly", + "meta/instanceID": "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390", + "_uuid": "f3d8dc65-91a6-4d0f-9e97-802128083390", + "_submission_time": "2013-02-14T15:37:22", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/ambulance": "True", + self.ambulance_key: "weekly", + "meta/instanceID": "uuid:9c6f3468-cfda-46e8-84c1-75458e72805d", + "_uuid": "9c6f3468-cfda-46e8-84c1-75458e72805d", + "_submission_time": "2013-02-14T15:37:23", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, + { + "available_transportation_types_to_referral_facility/taxi": "True", + "available_transportation_types_to_referral_facility/other": "True", + "available_transportation_types_to_referral_facility_other": "camel", + self.taxi_key: "daily", + "meta/instanceID": "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_uuid": "9f0a1508-c3b7-4c99-be00-9b237c26bcbf", + "_submission_time": "2013-02-14T15:37:24", + "_tags": "", + "_notes": "", + "_version": "2014111", + "_duration": "", + "_submitted_by": "bob", + "_total_media": "0", + "_media_count": "0", + "_media_all_received": "True", + }, ] dd = DataDictionary.objects.get(pk=self.xform.pk) - additional_headers = dd._additional_headers() + [ - '_id', '_date_modified'] + additional_headers = dd._additional_headers() + ["_id", "_date_modified"] for row, expected_dict in zip(actual_csv, data): test_dict = {} d = dict(zip(headers, row)) @@ -448,7 +504,7 @@ def _check_csv_export_second_pass(self): test_dict[k] = v this_list = [] for k, v in expected_dict.items(): - if k in ['image1', 'meta/instanceID'] or k.startswith("_"): + if k in ["image1", "meta/instanceID"] or k.startswith("_"): this_list.append((k, v)) else: this_list.append(("transport/" + k, v)) @@ -462,25 +518,31 @@ def test_xls_export_content(self): def _check_xls_export(self): xls_export_url = reverse( - 'xls_export', kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + "xls_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(xls_export_url) - expected_xls = openpyxl.open(filename=os.path.join( - self.this_directory, "fixtures", "transportation", - "transportation_export.xlsx"), data_only=True) + expected_xls = openpyxl.open( + filename=os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_export.xlsx", + ), + data_only=True, + ) content = get_response_content(response, decode=False) actual_xls = openpyxl.load_workbook(filename=BytesIO(content)) - actual_sheet = actual_xls.get_sheet_by_name('data') - expected_sheet = expected_xls.get_sheet_by_name('transportation') + actual_sheet = actual_xls.get_sheet_by_name("data") + expected_sheet = expected_xls.get_sheet_by_name("transportation") # check headers - self.assertEqual(list(actual_sheet.values)[0], - list(expected_sheet.values)[0]) + self.assertEqual(list(actual_sheet.values)[0], list(expected_sheet.values)[0]) # check cell data - self.assertEqual(len(list(actual_sheet.columns)), - len(list(expected_sheet.columns))) - self.assertEqual(len(list(actual_sheet.rows)), - len(list(expected_sheet.rows))) + self.assertEqual( + len(list(actual_sheet.columns)), len(list(expected_sheet.columns)) + ) + self.assertEqual(len(list(actual_sheet.rows)), len(list(expected_sheet.rows))) for i in range(1, len(list(actual_sheet.columns))): i = 1 actual_row = list(list(actual_sheet.values)[i]) @@ -497,10 +559,9 @@ def _check_delete(self): self.assertEquals(self.user.xforms.count(), 0) def test_405_submission(self): - url = reverse('submissions') + url = reverse("submissions") response = self.client.get(url) - self.assertContains( - response, 'Method "GET" not allowed', status_code=405) + self.assertContains(response, 'Method "GET" not allowed', status_code=405) def test_publish_bad_xls_with_unicode_in_error(self): """ @@ -510,24 +571,24 @@ def test_publish_bad_xls_with_unicode_in_error(self): """ self._create_user_and_login() path = os.path.join( - self.this_directory, 'fixtures', - 'form_with_unicode_in_relevant_column.xlsx') - with open(path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} - response = self.client.post('/%s/' % self.user.username, post_data) + self.this_directory, "fixtures", "form_with_unicode_in_relevant_column.xlsx" + ) + with open(path, "rb") as xls_file: + post_data = {"xls_file": xls_file} + response = self.client.post("/%s/" % self.user.username, post_data) self.assertEqual(response.status_code, 200) def test_metadata_file_hash(self): self._publish_transportation_form() - src = os.path.join(self.this_directory, "fixtures", - "transportation", "screenshot.png") - uf = UploadedFile(file=open(src, 'rb'), content_type='image/png') + src = os.path.join( + self.this_directory, "fixtures", "transportation", "screenshot.png" + ) + uf = UploadedFile(file=open(src, "rb"), content_type="image/png") count = MetaData.objects.count() MetaData.media_upload(self.xform, uf) # assert successful insert of new metadata record self.assertEqual(MetaData.objects.count(), count + 1) - md = MetaData.objects.get(object_id=self.xform.id, - data_value='screenshot.png') + md = MetaData.objects.get(object_id=self.xform.id, data_value="screenshot.png") # assert checksum string has been generated, hash length > 1 self.assertTrue(len(md.hash) > 16) @@ -537,13 +598,16 @@ def test_uuid_injection_in_cascading_select(self): """ pre_count = XForm.objects.count() xls_path = os.path.join( - self.this_directory, "fixtures", "cascading_selects", - "new_cascading_select.xlsx") + self.this_directory, + "fixtures", + "cascading_selects", + "new_cascading_select.xlsx", + ) file_name, file_ext = os.path.splitext(os.path.split(xls_path)[1]) TestBase._publish_xls_file(self, xls_path) post_count = XForm.objects.count() self.assertEqual(post_count, pre_count + 1) - xform = XForm.objects.latest('date_created') + xform = XForm.objects.latest("date_created") # check that the uuid is within the main instance/ # the one without an id attribute @@ -551,18 +615,24 @@ def test_uuid_injection_in_cascading_select(self): # check for instance nodes that are direct children of the model node model_node = xml.getElementsByTagName("model")[0] - instance_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName.lower() == "instance" and - not node.hasAttribute("id")] + instance_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName.lower() == "instance" + and not node.hasAttribute("id") + ] self.assertEqual(len(instance_nodes), 1) instance_node = instance_nodes[0] # get the first element whose id attribute is equal to our form's # id_string - form_nodes = [node for node in instance_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.getAttribute("id") == xform.id_string] + form_nodes = [ + node + for node in instance_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.getAttribute("id") == xform.id_string + ] form_node = form_nodes[0] # find the formhub node that has a uuid child node @@ -572,26 +642,31 @@ def test_uuid_injection_in_cascading_select(self): self.assertEqual(len(uuid_nodes), 1) # check for the calculate bind - calculate_bind_nodes = [node for node in model_node.childNodes if - node.nodeType == Node.ELEMENT_NODE and - node.tagName == "bind" and - node.getAttribute("nodeset") == - "/data/formhub/uuid"] + calculate_bind_nodes = [ + node + for node in model_node.childNodes + if node.nodeType == Node.ELEMENT_NODE + and node.tagName == "bind" + and node.getAttribute("nodeset") == "/data/formhub/uuid" + ] self.assertEqual(len(calculate_bind_nodes), 1) calculate_bind_node = calculate_bind_nodes[0] self.assertEqual( - calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid) + calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid + ) def test_csv_publishing(self): - csv_text = '\n'.join([ - 'survey,,', ',type,name,label', - ',text,whatsyourname,"What is your name?"', 'choices,,']) - url = reverse('user_profile', - kwargs={'username': self.user.username}) + csv_text = "\n".join( + [ + "survey,,", + ",type,name,label", + ',text,whatsyourname,"What is your name?"', + "choices,,", + ] + ) + url = reverse("user_profile", kwargs={"username": self.user.username}) num_xforms = XForm.objects.count() - params = { - 'text_xls_form': csv_text - } + params = {"text_xls_form": csv_text} self.response = self.client.post(url, params) self.assertEqual(XForm.objects.count(), num_xforms + 1) @@ -599,10 +674,9 @@ def test_truncate_xform_title_to_255(self): self._publish_transportation_form() title = "a" * (XFORM_TITLE_LENGTH + 1) groups = re.match( - r"(.+)([^<]+)(.*)", - self.xform.xml, re.DOTALL).groups() - self.xform.xml = "{0}{1}{2}".format( - groups[0], title, groups[2]) + r"(.+)([^<]+)(.*)", self.xform.xml, re.DOTALL + ).groups() + self.xform.xml = "{0}{1}{2}".format(groups[0], title, groups[2]) self.xform.title = title self.xform.save() self.assertEqual(self.xform.title, "a" * XFORM_TITLE_LENGTH) diff --git a/onadata/apps/main/tests/test_user_settings.py b/onadata/apps/main/tests/test_user_settings.py index 0869a1e8bd..9d91863090 100644 --- a/onadata/apps/main/tests/test_user_settings.py +++ b/onadata/apps/main/tests/test_user_settings.py @@ -1,5 +1,5 @@ from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.main.models import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -7,18 +7,18 @@ class TestUserSettings(TestBase): - def setUp(self): TestBase.setUp(self) self.settings_url = reverse( - profile_settings, kwargs={'username': self.user.username}) + profile_settings, kwargs={"username": self.user.username} + ) def test_render_user_settings(self): response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 200) def test_access_user_settings_non_owner(self): - self._create_user_and_login('alice', 'alice') + self._create_user_and_login("alice", "alice") response = self.client.get(self.settings_url) self.assertEqual(response.status_code, 404) @@ -31,14 +31,14 @@ def test_show_existing_profile_data(self): def test_update_user_settings(self): post_data = { - 'name': 'Bobby', - 'organization': 'Bob Inc', - 'city': 'Bobville', - 'country': 'BB', - 'twitter': 'bobo', - 'home_page': 'bob.com', - 'require_auth': True, - 'email': 'bob@bob.com' + "name": "Bobby", + "organization": "Bob Inc", + "city": "Bobville", + "country": "BB", + "twitter": "bobo", + "home_page": "bob.com", + "require_auth": True, + "email": "bob@bob.com", } response = self.client.post(self.settings_url, post_data) self.assertEqual(response.status_code, 302) @@ -47,7 +47,7 @@ def test_update_user_settings(self): try: self.assertEqual(self.user.profile.__dict__[key], value) except KeyError as e: - if key == 'email': + if key == "email": self.assertEqual(self.user.__dict__[key], value) else: raise e diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 95be00081c..24101e8980 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -3,7 +3,8 @@ RestService model """ import importlib -from future.utils import python_2_unicode_compatible + +from six import python_2_unicode_compatible from django.conf import settings from django.db import models @@ -23,30 +24,32 @@ class RestService(models.Model): """ class Meta: - app_label = 'restservice' - unique_together = ('service_url', 'xform', 'name') + app_label = "restservice" + unique_together = ("service_url", "xform", "name") service_url = models.URLField(ugettext_lazy("Service URL")) xform = models.ForeignKey(XForm, on_delete=models.CASCADE) name = models.CharField(max_length=50, choices=SERVICE_CHOICES) - date_created = models.DateTimeField( - auto_now_add=True, null=True, blank=True) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) - active = models.BooleanField(ugettext_lazy("Active"), default=True, - blank=False, null=False) - inactive_reason = models.TextField(ugettext_lazy("Inactive reason"), - blank=True, default="") + active = models.BooleanField( + ugettext_lazy("Active"), default=True, blank=False, null=False + ) + inactive_reason = models.TextField( + ugettext_lazy("Inactive reason"), blank=True, default="" + ) def __str__(self): - return u"%s:%s - %s" % (self.xform, self.long_name, self.service_url) + return "%s:%s - %s" % (self.xform, self.long_name, self.service_url) def get_service_definition(self): """ Returns ServiceDefinition class """ - services_to_modules = getattr(settings, 'REST_SERVICES_TO_MODULES', {}) + services_to_modules = getattr(settings, "REST_SERVICES_TO_MODULES", {}) module_name = services_to_modules.get( - self.name, 'onadata.apps.restservice.services.%s' % self.name) + self.name, "onadata.apps.restservice.services.%s" % self.name + ) module = importlib.import_module(module_name) return module.ServiceDefinition @@ -67,12 +70,11 @@ def delete_metadata(sender, instance, **kwargs): # pylint: disable=W0613 """ if instance.name in [TEXTIT, GOOGLE_SHEET]: MetaData.objects.filter( # pylint: disable=no-member - object_id=instance.xform.id, - data_type=instance.name).delete() + object_id=instance.xform.id, data_type=instance.name + ).delete() -post_delete.connect( - delete_metadata, sender=RestService, dispatch_uid='delete_metadata') +post_delete.connect(delete_metadata, sender=RestService, dispatch_uid="delete_metadata") # pylint: disable=W0613 @@ -80,19 +82,19 @@ def propagate_merged_datasets(sender, instance, **kwargs): """ Propagate the service to the individual forms of a merged dataset. """ - created = kwargs.get('created') + created = kwargs.get("created") if created and instance.xform.is_merged_dataset: for xform in instance.xform.mergedxform.xforms.all(): RestService.objects.create( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) post_save.connect( propagate_merged_datasets, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) # pylint: disable=W0613 @@ -104,9 +106,8 @@ def delete_merged_datasets_service(sender, instance, **kwargs): for xform in instance.xform.mergedxform.xforms.all(): try: service = RestService.objects.get( - service_url=instance.service_url, - xform=xform, - name=instance.name) + service_url=instance.service_url, xform=xform, name=instance.name + ) except RestService.DoesNotExist: pass else: @@ -116,4 +117,5 @@ def delete_merged_datasets_service(sender, instance, **kwargs): post_delete.connect( delete_merged_datasets_service, sender=RestService, - dispatch_uid='propagate_merged_datasets') + dispatch_uid="propagate_merged_datasets", +) diff --git a/onadata/apps/restservice/services/textit.py b/onadata/apps/restservice/services/textit.py index 3147e5075a..427372c0a0 100644 --- a/onadata/apps/restservice/services/textit.py +++ b/onadata/apps/restservice/services/textit.py @@ -1,6 +1,6 @@ import json import requests -from future.utils import iteritems +from six import iteritems from six import string_types from onadata.apps.main.models import MetaData @@ -11,7 +11,7 @@ class ServiceDefinition(RestServiceInterface): id = TEXTIT - verbose_name = u'TextIt POST' + verbose_name = "TextIt POST" def send(self, url, submission_instance): """ @@ -29,10 +29,12 @@ def send(self, url, submission_instance): post_data = { "extra": extra_data, "flow": flow, - "contacts": contacts.split(',') + "contacts": contacts.split(","), + } + headers = { + "Content-Type": "application/json", + "Authorization": "Token {}".format(token), } - headers = {"Content-Type": "application/json", - "Authorization": "Token {}".format(token)} requests.post(url, headers=headers, data=json.dumps(post_data)) @@ -48,14 +50,12 @@ def clean_keys_of_slashes(self, record): if not isinstance(value, string_types): record[key] = str(value) - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply # same - if value and isinstance(value, list)\ - and isinstance(value[0], dict): + if value and isinstance(value, list) and isinstance(value[0], dict): for v in value: self.clean_keys_of_slashes(v) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 2e314eb02e..b2a92961c2 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -13,13 +13,13 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ from floip import FloipSurvey from kombu.exceptions import OperationalError from pyxform.builder import create_survey_element_from_dict from pyxform.utils import has_external_choices from pyxform.xls2json import parse_file_to_json +from six import python_2_unicode_compatible from onadata.apps.logger.models.xform import XForm, check_version_set, check_xform_uuid from onadata.apps.logger.xform_instance_parser import XLSFormError diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 01f8dfde41..815d58b34c 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -3,7 +3,8 @@ Export model. """ import os -from future.utils import python_2_unicode_compatible + +from six import python_2_unicode_compatible from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class @@ -15,7 +16,7 @@ from onadata.libs.utils.common_tags import OSM from onadata.libs.utils import async_status -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" # pylint: disable=unused-argument @@ -23,7 +24,7 @@ def export_delete_callback(sender, **kwargs): """ Delete export file when an export object is deleted. """ - export = kwargs['instance'] + export = kwargs["instance"] storage = get_storage_class()() if export.filepath and storage.exists(export.filepath): storage.delete(export.filepath) @@ -38,7 +39,7 @@ def get_export_options_query_kwargs(options): if field in options: field_value = options.get(field) - key = 'options__{}'.format(field) + key = "options__{}".format(field) options_kwargs[key] = field_value return options_kwargs @@ -49,8 +50,9 @@ class ExportTypeError(Exception): """ ExportTypeError exception class. """ + def __str__(self): - return _(u'Invalid export type specified') + return _("Invalid export type specified") @python_2_unicode_compatible @@ -58,8 +60,9 @@ class ExportConnectionError(Exception): """ ExportConnectionError exception class. """ + def __str__(self): - return _(u'Export server is down.') + return _("Export server is down.") @python_2_unicode_compatible @@ -67,40 +70,41 @@ class Export(models.Model): """ Class representing a data export from an XForm """ - XLS_EXPORT = 'xls' - CSV_EXPORT = 'csv' - KML_EXPORT = 'kml' - ZIP_EXPORT = 'zip' - CSV_ZIP_EXPORT = 'csv_zip' - SAV_ZIP_EXPORT = 'sav_zip' - SAV_EXPORT = 'sav' - EXTERNAL_EXPORT = 'external' + + XLS_EXPORT = "xls" + CSV_EXPORT = "csv" + KML_EXPORT = "kml" + ZIP_EXPORT = "zip" + CSV_ZIP_EXPORT = "csv_zip" + SAV_ZIP_EXPORT = "sav_zip" + SAV_EXPORT = "sav" + EXTERNAL_EXPORT = "external" OSM_EXPORT = OSM - GOOGLE_SHEETS_EXPORT = 'gsheets' + GOOGLE_SHEETS_EXPORT = "gsheets" EXPORT_MIMES = { - 'xls': 'vnd.ms-excel', - 'xlsx': 'vnd.openxmlformats', - 'csv': 'csv', - 'zip': 'zip', - 'csv_zip': 'zip', - 'sav_zip': 'zip', - 'sav': 'sav', - 'kml': 'vnd.google-earth.kml+xml', - OSM: OSM + "xls": "vnd.ms-excel", + "xlsx": "vnd.openxmlformats", + "csv": "csv", + "zip": "zip", + "csv_zip": "zip", + "sav_zip": "zip", + "sav": "sav", + "kml": "vnd.google-earth.kml+xml", + OSM: OSM, } EXPORT_TYPES = [ - (XLS_EXPORT, 'Excel'), - (CSV_EXPORT, 'CSV'), - (ZIP_EXPORT, 'ZIP'), - (KML_EXPORT, 'kml'), - (CSV_ZIP_EXPORT, 'CSV ZIP'), - (SAV_ZIP_EXPORT, 'SAV ZIP'), - (SAV_EXPORT, 'SAV'), - (EXTERNAL_EXPORT, 'Excel'), + (XLS_EXPORT, "Excel"), + (CSV_EXPORT, "CSV"), + (ZIP_EXPORT, "ZIP"), + (KML_EXPORT, "kml"), + (CSV_ZIP_EXPORT, "CSV ZIP"), + (SAV_ZIP_EXPORT, "SAV ZIP"), + (SAV_EXPORT, "SAV"), + (EXTERNAL_EXPORT, "Excel"), (OSM, OSM), - (GOOGLE_SHEETS_EXPORT, 'Google Sheets'), + (GOOGLE_SHEETS_EXPORT, "Google Sheets"), ] EXPORT_OPTION_FIELDS = [ @@ -131,7 +135,7 @@ class Export(models.Model): MAX_EXPORTS = 10 # Required fields - xform = models.ForeignKey('logger.XForm', on_delete=models.CASCADE) + xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) export_type = models.CharField( max_length=10, choices=EXPORT_TYPES, default=XLS_EXPORT ) @@ -159,14 +163,15 @@ class Meta: unique_together = (("xform", "filename"),) def __str__(self): - return u'%s - %s (%s)' % (self.export_type, self.xform, self.filename) + return "%s - %s (%s)" % (self.export_type, self.xform, self.filename) def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.pk and self.xform: # if new, check if we've hit our limit for exports for this form, # if so, delete oldest num_existing_exports = Export.objects.filter( - xform=self.xform, export_type=self.export_type).count() + xform=self.xform, export_type=self.export_type + ).count() if num_existing_exports >= self.MAX_EXPORTS: Export._delete_oldest_export(self.xform, self.export_type) @@ -174,8 +179,7 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ # update time_of_last_submission with # xform.time_of_last_submission_update # pylint: disable=E1101 - self.time_of_last_submission = self.xform.\ - time_of_last_submission_update() + self.time_of_last_submission = self.xform.time_of_last_submission_update() if self.filename: self.internal_status = Export.SUCCESSFUL super(Export, self).save(*args, **kwargs) @@ -183,7 +187,8 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ @classmethod def _delete_oldest_export(cls, xform, export_type): oldest_export = Export.objects.filter( - xform=xform, export_type=export_type).order_by('created_on')[0] + xform=xform, export_type=export_type + ).order_by("created_on")[0] oldest_export.delete() @property @@ -227,9 +232,9 @@ def _update_filedir(self): if not self.filename: raise AssertionError() # pylint: disable=E1101 - self.filedir = os.path.join(self.xform.user.username, - 'exports', self.xform.id_string, - self.export_type) + self.filedir = os.path.join( + self.xform.user.username, "exports", self.xform.id_string, self.export_type + ) @property def filepath(self): @@ -273,16 +278,22 @@ def exports_outdated(cls, xform, export_type, options=None): try: export_options = get_export_options_query_kwargs(options) latest_export = Export.objects.filter( - xform=xform, export_type=export_type, + xform=xform, + export_type=export_type, internal_status__in=[Export.SUCCESSFUL, Export.PENDING], - **export_options).latest('created_on') + **export_options + ).latest("created_on") except cls.DoesNotExist: return True else: - if latest_export.time_of_last_submission is not None \ - and xform.time_of_last_submission_update() is not None: - return latest_export.time_of_last_submission <\ - xform.time_of_last_submission_update() + if ( + latest_export.time_of_last_submission is not None + and xform.time_of_last_submission_update() is not None + ): + return ( + latest_export.time_of_last_submission + < xform.time_of_last_submission_update() + ) # return true if we can't determine the status, to force # auto-generation @@ -293,8 +304,7 @@ def is_filename_unique(cls, xform, filename): """ Return True if the filename is unique. """ - return Export.objects.filter( - xform=xform, filename=filename).count() == 0 + return Export.objects.filter(xform=xform, filename=filename).count() == 0 post_delete.connect(export_delete_callback, sender=Export) diff --git a/onadata/apps/viewer/parsed_instance_tools.py b/onadata/apps/viewer/parsed_instance_tools.py index 9f9c179f7e..d166b6ed11 100644 --- a/onadata/apps/viewer/parsed_instance_tools.py +++ b/onadata/apps/viewer/parsed_instance_tools.py @@ -2,37 +2,35 @@ import six import datetime from builtins import str as text -from future.utils import iteritems from typing import Any, Tuple from onadata.libs.utils.common_tags import MONGO_STRFTIME, DATE_FORMAT -KNOWN_DATES = ['_submission_time'] +KNOWN_DATES = ["_submission_time"] NONE_JSON_FIELDS = { - '_submission_time': 'date_created', - '_date_modified': 'date_modified', - '_id': 'id', - '_version': 'version', - '_last_edited': 'last_edited' + "_submission_time": "date_created", + "_date_modified": "date_modified", + "_id": "id", + "_version": "version", + "_last_edited": "last_edited", } -def _json_sql_str(key, known_integers=None, known_dates=None, - known_decimals=None): +def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): if known_integers is None: known_integers = [] if known_dates is None: known_dates = [] if known_decimals is None: known_decimals = [] - _json_str = u"json->>%s" + _json_str = "json->>%s" if key in known_integers: - _json_str = u"CAST(json->>%s AS INT)" + _json_str = "CAST(json->>%s AS INT)" elif key in known_dates: - _json_str = u"CAST(json->>%s AS TIMESTAMP)" + _json_str = "CAST(json->>%s AS TIMESTAMP)" elif key in known_decimals: - _json_str = u"CAST(json->>%s AS DECIMAL)" + _json_str = "CAST(json->>%s AS DECIMAL)" return _json_str @@ -41,33 +39,25 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): # using a dictionary here just incase we will need to filter using # other table columns where, where_params = [], [] - OPERANDS = { - '$gt': '>', - '$gte': '>=', - '$lt': '<', - '$lte': '<=', - '$i': '~*' - } - for (field_key, field_value) in iteritems(query): + OPERANDS = {"$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<=", "$i": "~*"} + for (field_key, field_value) in six.iteritems(query): if isinstance(field_value, dict): if field_key in NONE_JSON_FIELDS: json_str = NONE_JSON_FIELDS.get(field_key) else: json_str = _json_sql_str( - field_key, known_integers, KNOWN_DATES, known_decimals) - for (key, value) in iteritems(field_value): + field_key, known_integers, KNOWN_DATES, known_decimals + ) + for (key, value) in six.iteritems(field_value): _v = None if key in OPERANDS: - where.append( - u' '.join([json_str, OPERANDS.get(key), u'%s']) - ) + where.append(" ".join([json_str, OPERANDS.get(key), "%s"])) _v = value if field_key in KNOWN_DATES: raw_date = value for date_format in (MONGO_STRFTIME, DATE_FORMAT): try: - _v = datetime.datetime.strptime(raw_date[:19], - date_format) + _v = datetime.datetime.strptime(raw_date[:19], date_format) except ValueError: pass if field_key in NONE_JSON_FIELDS: @@ -81,7 +71,7 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): elif field_value is None: where.append(f"json->>'{field_key}' IS NULL") else: - where.append(u"json->>%s = %s") + where.append("json->>%s = %s") where_params.extend((field_key, text(field_value))) return where + or_where, where_params + or_params @@ -102,55 +92,54 @@ def _merge_duplicate_keys(pairs: Tuple[str, Any]): return ret -def get_where_clause(query, form_integer_fields=None, - form_decimal_fields=None): +def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): if form_integer_fields is None: form_integer_fields = [] if form_decimal_fields is None: form_decimal_fields = [] - known_integers = ['_id'] + form_integer_fields + known_integers = ["_id"] + form_integer_fields known_decimals = form_decimal_fields where = [] where_params = [] try: if query and isinstance(query, (dict, six.string_types)): - query = query if isinstance(query, dict) else json.loads( - query, object_pairs_hook=_merge_duplicate_keys) + query = ( + query + if isinstance(query, dict) + else json.loads(query, object_pairs_hook=_merge_duplicate_keys) + ) or_where = [] or_params = [] if isinstance(query, list): query = query[0] - if isinstance(query, dict) and '$or' in list(query): - or_dict = query.pop('$or') + if isinstance(query, dict) and "$or" in list(query): + or_dict = query.pop("$or") for or_query in or_dict: for k, v in or_query.items(): if v is None: - or_where.extend( - [u"json->>'{}' IS NULL".format(k)]) + or_where.extend(["json->>'{}' IS NULL".format(k)]) elif isinstance(v, list): for value in v: or_where.extend(["json->>%s = %s"]) or_params.extend([k, value]) else: - or_where.extend( - [u"json->>%s = %s"]) + or_where.extend(["json->>%s = %s"]) or_params.extend([k, v]) - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] - where, where_params = _parse_where(query, known_integers, - known_decimals, or_where, - or_params) + where, where_params = _parse_where( + query, known_integers, known_decimals, or_where, or_params + ) except (ValueError, AttributeError) as e: - if query and isinstance(query, six.string_types) and \ - query.startswith('{'): + if query and isinstance(query, six.string_types) and query.startswith("{"): raise e # cast query param to text - where = [u"json::text ~* cast(%s as text)"] + where = ["json::text ~* cast(%s as text)"] where_params = [query] return where, where_params diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index 54a846e38c..1126caa81d 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -5,7 +5,7 @@ import sys from datetime import timedelta -from future.utils import iteritems +from six import iteritems from django.conf import settings from django.shortcuts import get_object_or_404 @@ -17,21 +17,23 @@ from onadata.apps.viewer.models.export import Export, ExportTypeError from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.common_tools import get_boolean_value, report_exception -from onadata.libs.utils.export_tools import (generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export) +from onadata.libs.utils.export_tools import ( + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, +) from onadata.celery import app -EXPORT_QUERY_KEY = 'query' +EXPORT_QUERY_KEY = "query" def _get_export_object(export_id): try: return Export.objects.get(id=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): + if getattr(settings, "SLAVE_DATABASES", []): from multidb.pinning import use_master with use_master: @@ -41,11 +43,7 @@ def _get_export_object(export_id): def _get_export_details(username, id_string, export_id): - details = { - 'export_id': export_id, - 'username': username, - 'id_string': id_string - } + details = {"export_id": export_id, "username": username, "id_string": id_string} return details @@ -65,24 +63,27 @@ def _create_export(xform, export_type, options): for (key, value) in iteritems(options) if key in Export.EXPORT_OPTION_FIELDS } - if query and 'query' not in export_options: - export_options['query'] = query + if query and "query" not in export_options: + export_options["query"] = query return Export.objects.create( - xform=xform, export_type=export_type, options=export_options) + xform=xform, export_type=export_type, options=export_options + ) export = _create_export(xform, export_type, options) result = None export_id = export.id - options.update({ - 'username': username, - 'id_string': id_string, - 'export_id': export_id, - 'query': query, - 'force_xlsx': force_xlsx - }) + options.update( + { + "username": username, + "id_string": id_string, + "export_id": export_id, + "query": query, + "force_xlsx": force_xlsx, + } + ) export_types = { Export.XLS_EXPORT: create_xls_export, @@ -93,7 +94,7 @@ def _create_export(xform, export_type, options): Export.ZIP_EXPORT: create_zip_export, Export.KML_EXPORT: create_kml_export, Export.OSM_EXPORT: create_osm_export, - Export.EXTERNAL_EXPORT: create_external_export + Export.EXTERNAL_EXPORT: create_external_export, } # start async export @@ -113,7 +114,7 @@ def _create_export(xform, export_type, options): # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save - if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False): + if getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False): export = get_object_or_404(Export, id=export.id) export.task_id = result.task_id export.save() @@ -129,7 +130,7 @@ def create_xls_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) - options["extension"] = 'xlsx' if force_xlsx else 'xls' + options["extension"] = "xlsx" if force_xlsx else "xls" try: export = _get_export_object(export_id) @@ -140,8 +141,9 @@ def create_xls_export(username, id_string, export_id, **options): # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: - gen_export = generate_export(Export.XLS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.XLS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -150,8 +152,10 @@ def create_xls_export(username, id_string, export_id, **options): report_exception( "XLS Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise @@ -171,8 +175,9 @@ def create_csv_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_EXPORT, export.xform, export_id, options + ) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records @@ -187,8 +192,10 @@ def create_csv_export(username, id_string, export_id, **options): report_exception( "CSV Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -212,7 +219,8 @@ def create_kml_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() @@ -220,8 +228,10 @@ def create_kml_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "KML Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -245,7 +255,8 @@ def create_osm_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -254,8 +265,10 @@ def create_osm_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -274,7 +287,8 @@ def create_zip_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -283,13 +297,17 @@ def create_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + ) raise else: if not settings.TESTING_MODE: delete_export.apply_async( - (), {'export_id': gen_export.id}, - countdown=settings.ZIP_EXPORT_COUNTDOWN) + (), + {"export_id": gen_export.id}, + countdown=settings.ZIP_EXPORT_COUNTDOWN, + ) return gen_export.id @@ -303,8 +321,9 @@ def create_csv_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.CSV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.CSV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) @@ -313,8 +332,10 @@ def create_csv_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "CSV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -330,8 +351,9 @@ def create_sav_zip_export(username, id_string, export_id, **options): try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.SAV_ZIP_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() @@ -339,8 +361,10 @@ def create_sav_zip_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -361,7 +385,8 @@ def create_external_export(username, id_string, export_id, **options): id_string, export_id, options, - xform=export.xform) + xform=export.xform, + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -369,8 +394,10 @@ def create_external_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "External Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -387,8 +414,9 @@ def create_google_sheet_export(username, id_string, export_id, **options): export = _get_export_object(export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery - gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, - export_id, options) + gen_export = generate_export( + Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options + ) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() @@ -396,8 +424,10 @@ def create_google_sheet_export(username, id_string, export_id, **options): details = _get_export_details(username, id_string, export_id) report_exception( "Google Export Exception: Export ID - " - "%(export_id)s, /%(username)s/%(id_string)s" % details, e, - sys.exc_info()) + "%(export_id)s, /%(username)s/%(id_string)s" % details, + e, + sys.exc_info(), + ) raise else: return gen_export.id @@ -427,7 +457,8 @@ def mark_expired_pending_exports_as_failed(): # pylint: disable=invalid-name task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.PENDING, created_on__lt=time_threshold) + internal_status=Export.PENDING, created_on__lt=time_threshold + ) exports.update(internal_status=Export.FAILED) @@ -439,5 +470,6 @@ def delete_expired_failed_exports(): task_lifespan = settings.EXPORT_TASK_LIFESPAN time_threshold = timezone.now() - timedelta(hours=task_lifespan) exports = Export.objects.filter( - internal_status=Export.FAILED, created_on__lt=time_threshold) + internal_status=Export.FAILED, created_on__lt=time_threshold + ) exports.delete() diff --git a/onadata/apps/viewer/tests/test_kml_export.py b/onadata/apps/viewer/tests/test_kml_export.py index c5573f5054..375b369568 100644 --- a/onadata/apps/viewer/tests/test_kml_export.py +++ b/onadata/apps/viewer/tests/test_kml_export.py @@ -1,7 +1,7 @@ import os from django.urls import reverse -from future.utils import iteritems +from six import iteritems from onadata.apps.logger.models.instance import Instance from onadata.apps.main.tests.test_base import TestBase @@ -9,43 +9,46 @@ class TestKMLExport(TestBase): - def _publish_survey(self): self.this_directory = os.path.dirname(__file__) xls_path = self._fixture_path("gps", "gps.xlsx") TestBase._publish_xls_file(self, xls_path) def test_kml_export(self): - id_string = 'gps' + id_string = "gps" self._publish_survey() self._make_submissions_gps() - self.fixtures = os.path.join( - self.this_directory, 'fixtures', 'kml_export') + self.fixtures = os.path.join(self.this_directory, "fixtures", "kml_export") url = reverse( - kml_export, - kwargs={'username': self.user.username, 'id_string': id_string}) + kml_export, kwargs={"username": self.user.username, "id_string": id_string} + ) response = self.client.get(url) instances = Instance.objects.filter( - xform__user=self.user, xform__id_string=id_string, - geom__isnull=False - ).order_by('id') + xform__user=self.user, xform__id_string=id_string, geom__isnull=False + ).order_by("id") self.assertEqual(instances.count(), 2) # create a tuple of replacement data per instance - replacement_data = [["{:,}".format(x) for x in [ - i.pk, i.point.x, i.point.y]] for i in instances] + replacement_data = [ + ["{:,}".format(x) for x in [i.pk, i.point.x, i.point.y]] for i in instances + ] # assuming 2 instances, flatten and assign to template names - replacement_dict = dict(zip(['pk1', 'x1', 'y1', 'pk2', 'x2', 'y2'], - [i for s in replacement_data for i in s])) - - with open(os.path.join(self.fixtures, 'export.kml')) as f: + replacement_dict = dict( + zip( + ["pk1", "x1", "y1", "pk2", "x2", "y2"], + [i for s in replacement_data for i in s], + ) + ) + + with open(os.path.join(self.fixtures, "export.kml")) as f: expected_content = f.read() for (template_name, template_data) in iteritems(replacement_dict): expected_content = expected_content.replace( - '{{%s}}' % template_name, template_data) + "{{%s}}" % template_name, template_data + ) self.assertMultiLineEqual( - expected_content.strip(), - response.content.decode('utf-8').strip()) + expected_content.strip(), response.content.decode("utf-8").strip() + ) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 297f7de322..fe6711a66b 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -13,30 +13,28 @@ from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator -from future.utils import iteritems +from fsix import iteritems from rest_framework import negotiation -from rest_framework.renderers import (BaseRenderer, JSONRenderer, - StaticHTMLRenderer, TemplateHTMLRenderer) +from rest_framework.renderers import ( + BaseRenderer, + JSONRenderer, + StaticHTMLRenderer, + TemplateHTMLRenderer, +) from rest_framework.utils.encoders import JSONEncoder from rest_framework_xml.renderers import XMLRenderer from onadata.libs.utils.osm import get_combined_osm IGNORE_FIELDS = [ - 'formhub/uuid', - 'meta/contactID', - 'meta/deprecatedID', - 'meta/instanceID', - 'meta/sessionID', + "formhub/uuid", + "meta/contactID", + "meta/deprecatedID", + "meta/instanceID", + "meta/sessionID", ] -FORMLIST_MANDATORY_FIELDS = [ - 'formID', - 'name', - 'version', - 'hash', - 'downloadUrl' -] +FORMLIST_MANDATORY_FIELDS = ["formID", "name", "version", "hash", "downloadUrl"] def pairing(val1, val2): @@ -52,16 +50,19 @@ def floip_rows_list(data): """ Yields a row of FLOIP results data from dict data. """ - _submission_time = pytz.timezone('UTC').localize( - parse_datetime(data['_submission_time'])).isoformat() + _submission_time = ( + pytz.timezone("UTC") + .localize(parse_datetime(data["_submission_time"])) + .isoformat() + ) for i, key in enumerate(data, 1): - if not (key.startswith('_') or key in IGNORE_FIELDS): - instance_id = data['_id'] + if not (key.startswith("_") or key in IGNORE_FIELDS): + instance_id = data["_id"] yield [ _submission_time, # Timestamp int(pairing(instance_id, i)), # Row ID - data.get('meta/contactID', data.get('_submitted_by')), - data.get('meta/sessionID') or data.get('_uuid') or instance_id, + data.get("meta/contactID", data.get("_submitted_by")), + data.get("meta/sessionID") or data.get("_uuid") or instance_id, key, # Question ID data[key], # Response None, # Response Metadata @@ -94,13 +95,14 @@ class XLSRenderer(BaseRenderer): # pylint: disable=R0903 XLSRenderer - renders .xls spreadsheet documents with application/vnd.openxmlformats. """ - media_type = 'application/vnd.openxmlformats' - format = 'xls' + + media_type = "application/vnd.openxmlformats" + format = "xls" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data @@ -109,29 +111,32 @@ class XLSXRenderer(XLSRenderer): # pylint: disable=too-few-public-methods XLSRenderer - renders .xlsx spreadsheet documents with application/vnd.openxmlformats. """ - format = 'xlsx' + + format = "xlsx" class CSVRenderer(BaseRenderer): # pylint: disable=abstract-method, R0903 """ XLSRenderer - renders comma separated files (CSV) with text/csv. """ - media_type = 'text/csv' - format = 'csv' - charset = 'utf-8' + + media_type = "text/csv" + format = "csv" + charset = "utf-8" class CSVZIPRenderer(BaseRenderer): # pylint: disable=R0903 """ CSVZIPRenderer - renders a ZIP file that contains CSV files. """ - media_type = 'application/octet-stream' - format = 'csvzip' + + media_type = "application/octet-stream" + format = "csvzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -141,13 +146,14 @@ class SAVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SAVZIPRenderer - renders a ZIP file that contains SPSS SAV files. """ - media_type = 'application/octet-stream' - format = 'savzip' + + media_type = "application/octet-stream" + format = "savzip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -157,9 +163,10 @@ class SurveyRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ SurveyRenderer - renders XML data. """ - media_type = 'application/xml' - format = 'xml' - charset = 'utf-8' + + media_type = "application/xml" + format = "xml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data @@ -169,9 +176,10 @@ class KMLRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ KMLRenderer - renders KML XML data. """ - media_type = 'application/xml' - format = 'kml' - charset = 'utf-8' + + media_type = "application/xml" + format = "kml" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return data @@ -181,7 +189,8 @@ class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=R0903 """ GoogleSheetsRenderer = Google Sheets excel exports. """ - format = 'gsheets' + + format = "gsheets" class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): @@ -196,9 +205,7 @@ def filter_renderers(self, renderers, format): # pylint: disable=W0622 so that we only negotiation against those that accept that format. If there is no renderer available, we use MediaFileRenderer. """ - renderers = [ - renderer for renderer in renderers if renderer.format == format - ] + renderers = [renderer for renderer in renderers if renderer.format == format] if not renderers: renderers = [MediaFileRenderer()] @@ -209,14 +216,15 @@ class MediaFileRenderer(BaseRenderer): # pylint: disable=R0903 """ MediaFileRenderer - render binary media files. """ - media_type = '*/*' + + media_type = "*/*" format = None charset = None - render_style = 'binary' + render_style = "binary" def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") return data @@ -225,11 +233,11 @@ class XFormListRenderer(BaseRenderer): # pylint: disable=R0903 Renderer which serializes to XML. """ - media_type = 'text/xml' - format = 'xml' - charset = 'utf-8' - root_node = 'xforms' - element_node = 'xform' + media_type = "text/xml" + format = "xml" + charset = "utf-8" + root_node = "xforms" + element_node = "xform" xmlns = "http://openrosa.org/xforms/xformsList" def render(self, data, accepted_media_type=None, renderer_context=None): @@ -237,7 +245,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): Renders *obj* into serialized XML. """ if data is None: - return '' + return "" elif isinstance(data, six.string_types): return data @@ -245,7 +253,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement(self.root_node, {'xmlns': self.xmlns}) + xml.startElement(self.root_node, {"xmlns": self.xmlns}) self._to_xml(xml, data) @@ -281,6 +289,7 @@ class XFormManifestRenderer(XFormListRenderer): # pylint: disable=R0903 """ XFormManifestRenderer - render XFormManifest XML. """ + root_node = "manifest" element_node = "mediaFile" xmlns = "http://openrosa.org/xforms/xformsManifest" @@ -290,30 +299,32 @@ class TemplateXMLRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 """ TemplateXMLRenderer - Render XML template. """ - format = 'xml' - media_type = 'text/xml' + + format = "xml" + media_type = "text/xml" def render(self, data, accepted_media_type=None, renderer_context=None): renderer_context = renderer_context or {} - response = renderer_context['response'] + response = renderer_context["response"] if response and response.exception: - return XMLRenderer().render(data, accepted_media_type, - renderer_context) + return XMLRenderer().render(data, accepted_media_type, renderer_context) - return super(TemplateXMLRenderer, - self).render(data, accepted_media_type, renderer_context) + return super(TemplateXMLRenderer, self).render( + data, accepted_media_type, renderer_context + ) class InstanceXMLRenderer(XMLRenderer): """ InstanceXMLRenderer - Renders Instance XML """ - root_tag_name = 'submission-batch' - item_tag_name = 'submission-item' + + root_tag_name = "submission-batch" + item_tag_name = "submission-item" def _get_current_buffer_data(self): - if hasattr(self, 'stream'): + if hasattr(self, "stream"): ret = self.stream.getvalue() self.stream.truncate(0) self.stream.seek(0) @@ -327,10 +338,7 @@ def stream_data(self, data, serializer): xml = SimplerXMLGenerator(self.stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()} - ) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) yield self._get_current_buffer_data() @@ -376,9 +384,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - xml.startElement( - self.root_tag_name, - {'serverTime': timezone.now().isoformat()}) + xml.startElement(self.root_tag_name, {"serverTime": timezone.now().isoformat()}) self._to_xml(xml, data) @@ -392,8 +398,8 @@ def _pop_xml_attributes(self, xml_dictionary: dict) -> Tuple[dict, dict]: attributes = {} for key, value in xml_dictionary.items(): - if key.startswith('@'): - attributes.update({key.replace('@', ''): value}) + if key.startswith("@"): + attributes.update({key.replace("@", ""): value}) del ret[key] return ret, attributes @@ -437,17 +443,19 @@ class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=R0903 """ StaticXMLRenderer - render static XML document. """ - format = 'xml' - media_type = 'text/xml' + + format = "xml" + media_type = "text/xml" class GeoJsonRenderer(BaseRenderer): # pylint: disable=R0903 """ GeoJsonRenderer - render .geojson data as json. """ - media_type = 'application/json' - format = 'geojson' - charset = 'utf-8' + + media_type = "application/json" + format = "geojson" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): return json.dumps(data) @@ -457,15 +465,16 @@ class OSMRenderer(BaseRenderer): # pylint: disable=R0903 """ OSMRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + + media_type = "text/xml" + format = "osm" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): # Process error before making a list if isinstance(data, dict): - if 'detail' in data: - return u'' + data['detail'] + '' + if "detail" in data: + return "" + data["detail"] + "" # Combine/concatenate the list of osm files to one file def _list(list_or_item): @@ -483,43 +492,47 @@ class OSMExportRenderer(BaseRenderer): # pylint: disable=R0903, W0223 """ OSMExportRenderer - render .osm data as XML. """ - media_type = 'text/xml' - format = 'osm' - charset = 'utf-8' + + media_type = "text/xml" + format = "osm" + charset = "utf-8" class DebugToolbarRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 """ DebugToolbarRenderer - render .debug as HTML. """ - media_type = 'text/html' - charset = 'utf-8' - format = 'debug' - template_name = 'debug.html' + + media_type = "text/html" + charset = "utf-8" + format = "debug" + template_name = "debug.html" def render(self, data, accepted_media_type=None, renderer_context=None): data = { - 'debug_data': - str( + "debug_data": str( JSONRenderer().render(data, renderer_context=renderer_context), - self.charset) + self.charset, + ) } return super(DebugToolbarRenderer, self).render( - data, accepted_media_type, renderer_context) + data, accepted_media_type, renderer_context + ) class ZipRenderer(BaseRenderer): # pylint: disable=R0903 """ ZipRenderer - render .zip files. """ - media_type = 'application/octet-stream' - format = 'zip' + + media_type = "application/octet-stream" + format = "zip" charset = None def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): - return data.encode('utf-8') + return data.encode("utf-8") elif isinstance(data, dict): return json.dumps(data) return data @@ -529,6 +542,7 @@ class DecimalJSONRenderer(JSONRenderer): """ Extends the default json renderer to handle Decimal('NaN') values """ + encoder_class = DecimalEncoder @@ -536,19 +550,21 @@ class FLOIPRenderer(JSONRenderer): """ FLOIP Results data renderer. """ - media_type = 'application/vnd.org.flowinterop.results+json' - format = 'json' - charset = 'utf-8' + + media_type = "application/vnd.org.flowinterop.results+json" + format = "json" + charset = "utf-8" def render(self, data, accepted_media_type=None, renderer_context=None): - request = renderer_context['request'] - response = renderer_context['response'] + request = renderer_context["request"] + response = renderer_context["response"] results = data - if request.method == 'GET' and response.status_code == 200: + if request.method == "GET" and response.status_code == 200: if isinstance(data, dict): results = [i for i in floip_rows_list(data)] else: results = [i for i in floip_list(data)] - return super(FLOIPRenderer, self).render(results, accepted_media_type, - renderer_context) + return super(FLOIPRenderer, self).render( + results, accepted_media_type, renderer_context + ) diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index d59ecdd959..9459353635 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -1,5 +1,5 @@ import json -from future.utils import listvalues +from six import itervalues from rest_framework import serializers @@ -13,7 +13,7 @@ def dict_key_for_value(_dict, value): """ This function is used to get key by value in a dictionary """ - return list(_dict)[listvalues(_dict).index(value)] + return list(_dict)[list(itervalues(_dict)).index(value)] def get_path(data, question_name, path_list): @@ -25,12 +25,12 @@ def get_path(data, question_name, path_list): :return: an xpath which is a string or None if name cannot be found :rtype: string or None """ - name = data.get('name') + name = data.get("name") if name == question_name: - return '/'.join(path_list) - elif data.get('children') is not None: - for node in data.get('children'): - path_list.append(node.get('name')) + return "/".join(path_list) + elif data.get("children") is not None: + for node in data.get("children"): + path_list.append(node.get("name")) path = get_path(node, question_name, path_list) if path is not None: return path @@ -40,26 +40,35 @@ def get_path(data, question_name, path_list): class AttachmentSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='attachment-detail', - lookup_field='pk') + url = serializers.HyperlinkedIdentityField( + view_name="attachment-detail", lookup_field="pk" + ) field_xpath = serializers.SerializerMethodField() download_url = serializers.SerializerMethodField() small_download_url = serializers.SerializerMethodField() medium_download_url = serializers.SerializerMethodField() - xform = serializers.ReadOnlyField(source='instance.xform.pk') - instance = serializers.PrimaryKeyRelatedField( - queryset=Instance.objects.all()) - filename = serializers.ReadOnlyField(source='media_file.name') + xform = serializers.ReadOnlyField(source="instance.xform.pk") + instance = serializers.PrimaryKeyRelatedField(queryset=Instance.objects.all()) + filename = serializers.ReadOnlyField(source="media_file.name") class Meta: - fields = ('url', 'filename', 'mimetype', 'field_xpath', 'id', 'xform', - 'instance', 'download_url', 'small_download_url', - 'medium_download_url') + fields = ( + "url", + "filename", + "mimetype", + "field_xpath", + "id", + "xform", + "instance", + "download_url", + "small_download_url", + "medium_download_url", + ) model = Attachment @check_obj def get_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") if obj: path = get_attachment_url(obj) @@ -67,18 +76,18 @@ def get_download_url(self, obj): return request.build_absolute_uri(path) if request else path def get_small_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'small') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "small") return request.build_absolute_uri(path) if request else path def get_medium_download_url(self, obj): - request = self.context.get('request') + request = self.context.get("request") - if obj.mimetype.startswith('image'): - path = get_attachment_url(obj, 'medium') + if obj.mimetype.startswith("image"): + path = get_attachment_url(obj, "medium") return request.build_absolute_uri(path) if request else path diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 5444c0b552..967dbc56bb 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -13,7 +13,7 @@ from django.db.utils import IntegrityError from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from rest_framework import serializers from rest_framework.reverse import reverse @@ -21,42 +21,45 @@ from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models import MetaData from onadata.libs.permissions import ROLES, ManagerRole -from onadata.libs.serializers.fields.instance_related_field import \ - InstanceRelatedField -from onadata.libs.serializers.fields.project_related_field import \ - ProjectRelatedField -from onadata.libs.serializers.fields.xform_related_field import \ - XFormRelatedField +from onadata.libs.serializers.fields.instance_related_field import InstanceRelatedField +from onadata.libs.serializers.fields.project_related_field import ProjectRelatedField +from onadata.libs.serializers.fields.xform_related_field import XFormRelatedField from onadata.libs.utils.common_tags import ( - XFORM_META_PERMS, SUBMISSION_REVIEW, IMPORTED_VIA_CSV_BY) + XFORM_META_PERMS, + SUBMISSION_REVIEW, + IMPORTED_VIA_CSV_BY, +) -UNIQUE_TOGETHER_ERROR = u"Object already exists" +UNIQUE_TOGETHER_ERROR = "Object already exists" -CSV_CONTENT_TYPE = 'text/csv' -MEDIA_TYPE = 'media' -DOC_TYPE = 'supporting_doc' +CSV_CONTENT_TYPE = "text/csv" +MEDIA_TYPE = "media" +DOC_TYPE = "supporting_doc" METADATA_TYPES = ( - ('data_license', _(u"Data License")), - ('enketo_preview_url', _(u"Enketo Preview URL")), - ('enketo_url', _(u"Enketo URL")), - ('form_license', _(u"Form License")), - ('mapbox_layer', _(u"Mapbox Layer")), - (MEDIA_TYPE, _(u"Media")), - ('public_link', _(u"Public Link")), - ('source', _(u"Source")), - (DOC_TYPE, _(u"Supporting Document")), - ('external_export', _(u"External Export")), - ('textit', _(u"TextIt")), - ('google_sheets', _(u"Google Sheet")), - ('xform_meta_perms', _("Xform meta permissions")), - ('submission_review', _("Submission Review")), - (IMPORTED_VIA_CSV_BY, _("Imported via CSV by"))) # yapf:disable - -DATAVIEW_TAG = 'dataview' -XFORM_TAG = 'xform' - -PROJECT_METADATA_TYPES = ((MEDIA_TYPE, _(u"Media")), - ('supporting_doc', _(u"Supporting Document"))) + ("data_license", _("Data License")), + ("enketo_preview_url", _("Enketo Preview URL")), + ("enketo_url", _("Enketo URL")), + ("form_license", _("Form License")), + ("mapbox_layer", _("Mapbox Layer")), + (MEDIA_TYPE, _("Media")), + ("public_link", _("Public Link")), + ("source", _("Source")), + (DOC_TYPE, _("Supporting Document")), + ("external_export", _("External Export")), + ("textit", _("TextIt")), + ("google_sheets", _("Google Sheet")), + ("xform_meta_perms", _("Xform meta permissions")), + ("submission_review", _("Submission Review")), + (IMPORTED_VIA_CSV_BY, _("Imported via CSV by")), +) # yapf:disable + +DATAVIEW_TAG = "dataview" +XFORM_TAG = "xform" + +PROJECT_METADATA_TYPES = ( + (MEDIA_TYPE, _("Media")), + ("supporting_doc", _("Supporting Document")), +) def get_linked_object(parts): @@ -73,12 +76,14 @@ def get_linked_object(parts): try: obj_pk = int(obj_pk) except ValueError: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid %(type)s id %(id)s." % - {'type': obj_type, - 'id': obj_pk}) - }) + raise serializers.ValidationError( + { + "data_value": _( + "Invalid %(type)s id %(id)s." + % {"type": obj_type, "id": obj_pk} + ) + } + ) else: model = DataView if obj_type == DATAVIEW_TAG else XForm @@ -89,17 +94,17 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): """ MetaData HyperlinkedModelSerializer """ + id = serializers.ReadOnlyField() # pylint: disable=C0103 xform = XFormRelatedField(queryset=XForm.objects.all(), required=False) - project = ProjectRelatedField( - queryset=Project.objects.all(), required=False) - instance = InstanceRelatedField( - queryset=Instance.objects.all(), required=False) + project = ProjectRelatedField(queryset=Project.objects.all(), required=False) + instance = InstanceRelatedField(queryset=Instance.objects.all(), required=False) data_value = serializers.CharField(max_length=255, required=True) data_type = serializers.ChoiceField(choices=METADATA_TYPES) data_file = serializers.FileField(required=False) data_file_type = serializers.CharField( - max_length=255, required=False, allow_blank=True) + max_length=255, required=False, allow_blank=True + ) media_url = serializers.SerializerMethodField() date_created = serializers.ReadOnlyField() @@ -108,102 +113,133 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = MetaData - fields = ('id', 'xform', 'project', 'instance', 'data_value', - 'data_type', 'data_file', 'data_file_type', 'media_url', - 'file_hash', 'url', 'date_created') + fields = ( + "id", + "xform", + "project", + "instance", + "data_value", + "data_type", + "data_file", + "data_file_type", + "media_url", + "file_hash", + "url", + "date_created", + ) def get_media_url(self, obj): """ Returns media URL for given metadata """ - if obj.data_type in [DOC_TYPE, MEDIA_TYPE] and\ - getattr(obj, "data_file") and getattr(obj.data_file, "url"): + if ( + obj.data_type in [DOC_TYPE, MEDIA_TYPE] + and getattr(obj, "data_file") + and getattr(obj.data_file, "url") + ): return obj.data_file.url elif obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: kwargs = { - 'kwargs': { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "kwargs": { + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, }, - 'request': self.context.get('request'), - 'format': 'csv' + "request": self.context.get("request"), + "format": "csv", } - return reverse('xform-media', **kwargs) + return reverse("xform-media", **kwargs) def validate(self, attrs): """ Validate url if we are adding a media uri instead of a media file """ - value = attrs.get('data_value') - data_type = attrs.get('data_type') - data_file = attrs.get('data_file') - - if not ('project' in attrs or 'xform' in attrs or 'instance' in attrs): - raise serializers.ValidationError({ - 'missing_field': - _(u"`xform` or `project` or `instance`" - "field is required.") - }) + value = attrs.get("data_value") + data_type = attrs.get("data_type") + data_file = attrs.get("data_file") + + if not ("project" in attrs or "xform" in attrs or "instance" in attrs): + raise serializers.ValidationError( + { + "missing_field": _( + "`xform` or `project` or `instance`" "field is required." + ) + } + ) if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES - data_content_type = data_file.content_type \ - if data_file.content_type in allowed_types else \ - mimetypes.guess_type(data_file.name)[0] + data_content_type = ( + data_file.content_type + if data_file.content_type in allowed_types + else mimetypes.guess_type(data_file.name)[0] + ) if data_content_type not in allowed_types: - raise serializers.ValidationError({ - 'data_file': - _('Unsupported media file type %s' % data_content_type)}) + raise serializers.ValidationError( + { + "data_file": _( + "Unsupported media file type %s" % data_content_type + ) + } + ) else: - attrs['data_file_type'] = data_content_type + attrs["data_file_type"] = data_content_type - if data_type == 'media' and data_file is None: + if data_type == "media" and data_file is None: try: URLValidator()(value) except ValidationError: parts = value.split() if len(parts) < 3: - raise serializers.ValidationError({ - 'data_value': - _(u"Expecting 'xform [xform id] [media name]' " - "or 'dataview [dataview id] [media name]' " - "or a valid URL.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "Expecting 'xform [xform id] [media name]' " + "or 'dataview [dataview id] [media name]' " + "or a valid URL." + ) + } + ) obj = get_linked_object(parts) if obj: xform = obj.xform if isinstance(obj, DataView) else obj - request = self.context['request'] + request = self.context["request"] user_has_role = ManagerRole.user_has_role - has_perm = user_has_role(request.user, xform) or \ - user_has_role(request.user, obj.project) + has_perm = user_has_role(request.user, xform) or user_has_role( + request.user, obj.project + ) if not has_perm: - raise serializers.ValidationError({ - 'data_value': - _(u"User has no permission to " - "the dataview.") - }) + raise serializers.ValidationError( + { + "data_value": _( + "User has no permission to " "the dataview." + ) + } + ) else: - raise serializers.ValidationError({ - 'data_value': - _(u"Invalid url '%s'." % value) - }) + raise serializers.ValidationError( + {"data_value": _("Invalid url '%s'." % value)} + ) else: # check if we have a value for the filename. if not os.path.basename(urlparse(value).path): - raise serializers.ValidationError({ - 'data_value': - _(u"Cannot get filename from URL %s. URL should " - u"include the filename e.g " - u"http://example.com/data.csv" % value) - }) + raise serializers.ValidationError( + { + "data_value": _( + "Cannot get filename from URL %s. URL should " + "include the filename e.g " + "http://example.com/data.csv" % value + ) + } + ) if data_type == XFORM_META_PERMS: - perms = value.split('|') + perms = value.split("|") if len(perms) != 2 or not set(perms).issubset(set(ROLES)): raise serializers.ValidationError( - _(u"Format 'role'|'role' or Invalid role")) + _("Format 'role'|'role' or Invalid role") + ) return attrs @@ -215,34 +251,38 @@ def get_content_object(self, validated_data): """ if validated_data: - return (validated_data.get('xform') or - validated_data.get('project') or - validated_data.get('instance')) + return ( + validated_data.get("xform") + or validated_data.get("project") + or validated_data.get("instance") + ) def create(self, validated_data): - data_type = validated_data.get('data_type') - data_file = validated_data.get('data_file') - data_file_type = validated_data.get('data_file_type') + data_type = validated_data.get("data_type") + data_file = validated_data.get("data_file") + data_file_type = validated_data.get("data_file_type") content_object = self.get_content_object(validated_data) - data_value = data_file.name \ - if data_file else validated_data.get('data_value') + data_value = data_file.name if data_file else validated_data.get("data_value") # not exactly sure what changed in the requests.FILES for django 1.7 # csv files uploaded in windows do not have the text/csv content_type # this works around that - if data_type == MEDIA_TYPE and data_file \ - and data_file.name.lower().endswith('.csv') \ - and data_file_type != CSV_CONTENT_TYPE: + if ( + data_type == MEDIA_TYPE + and data_file + and data_file.name.lower().endswith(".csv") + and data_file_type != CSV_CONTENT_TYPE + ): data_file_type = CSV_CONTENT_TYPE content_type = ContentType.objects.get_for_model(content_object) try: if data_type == XFORM_META_PERMS: - metadata = \ - MetaData.xform_meta_permission(content_object, - data_value=data_value) + metadata = MetaData.xform_meta_permission( + content_object, data_value=data_value + ) update_role_by_meta_xform_perms(content_object) elif data_type == SUBMISSION_REVIEW: @@ -251,10 +291,12 @@ def create(self, validated_data): raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) else: metadata = MetaData.submission_review( - content_object, data_value=data_value) + content_object, data_value=data_value + ) elif data_type == IMPORTED_VIA_CSV_BY: metadata = MetaData.instance_csv_imported_by( - content_object, data_value=data_value) + content_object, data_value=data_value + ) else: metadata = MetaData.objects.create( content_type=content_type, @@ -262,15 +304,15 @@ def create(self, validated_data): data_value=data_value, data_file=data_file, data_file_type=data_file_type, - object_id=content_object.id) + object_id=content_object.id, + ) return metadata except IntegrityError: raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) def update(self, instance, validated_data): - instance = super(MetaDataSerializer, self).update( - instance, validated_data) + instance = super(MetaDataSerializer, self).update(instance, validated_data) if instance.data_type == XFORM_META_PERMS: update_role_by_meta_xform_perms(instance.content_object) diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index ce292cf95e..1eaeb47fd4 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -1,5 +1,5 @@ from builtins import bytes as b -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from django.contrib.auth.models import User from django.contrib.auth.tokens import PasswordResetTokenGenerator @@ -16,44 +16,55 @@ class CustomPasswordResetTokenGenerator(PasswordResetTokenGenerator): """Custom Password Token Generator Class.""" + def _make_hash_value(self, user, timestamp): # Include user email alongside user password to the generated token # as the user state object that might change after a password reset # to produce a token that invalidated. - login_timestamp = '' if user.last_login is None\ + login_timestamp = ( + "" + if user.last_login is None else user.last_login.replace(microsecond=0, tzinfo=None) - return str(user.pk) + user.password + user.email +\ - str(login_timestamp) + str(timestamp) + ) + return ( + str(user.pk) + + user.password + + user.email + + str(login_timestamp) + + str(timestamp) + ) default_token_generator = CustomPasswordResetTokenGenerator() -def get_password_reset_email(user, reset_url, - subject_template_name='registration/password_reset_subject.txt', # noqa - email_template_name='api_password_reset_email.html', # noqa - token_generator=default_token_generator, - email_subject=None): +def get_password_reset_email( + user, + reset_url, + subject_template_name="registration/password_reset_subject.txt", # noqa + email_template_name="api_password_reset_email.html", # noqa + token_generator=default_token_generator, + email_subject=None, +): """Creates the subject and email body for password reset email.""" result = urlparse(reset_url) site_name = domain = result.hostname - encoded_username = urlsafe_base64_encode(b(user.username.encode('utf-8'))) + encoded_username = urlsafe_base64_encode(b(user.username.encode("utf-8"))) c = { - 'email': user.email, - 'domain': domain, - 'path': result.path, - 'site_name': site_name, - 'uid': urlsafe_base64_encode(force_bytes(user.pk)), - 'username': user.username, - 'encoded_username': encoded_username, - 'token': token_generator.make_token(user), - 'protocol': result.scheme if result.scheme != '' else 'http', + "email": user.email, + "domain": domain, + "path": result.path, + "site_name": site_name, + "uid": urlsafe_base64_encode(force_bytes(user.pk)), + "username": user.username, + "encoded_username": encoded_username, + "token": token_generator.make_token(user), + "protocol": result.scheme if result.scheme != "" else "http", } # if subject email provided don't load the subject template - subject = email_subject or loader.render_to_string(subject_template_name, - c) + subject = email_subject or loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines - subject = ''.join(subject.splitlines()) + subject = "".join(subject.splitlines()) email = loader.render_to_string(email_template_name, c) return subject, email @@ -66,7 +77,7 @@ def get_user_from_uid(uid): uid = urlsafe_base64_decode(uid) user = User.objects.get(pk=uid) except (TypeError, ValueError, OverflowError, User.DoesNotExist): - raise serializers.ValidationError(_(u"Invalid uid %s") % uid) + raise serializers.ValidationError(_("Invalid uid %s") % uid) return user @@ -91,11 +102,13 @@ def __init__(self, email, reset_url, email_subject=None): self.reset_url = reset_url self.email_subject = email_subject - def save(self, - subject_template_name='registration/password_reset_subject.txt', - email_template_name='api_password_reset_email.html', - token_generator=default_token_generator, - from_email=None): + def save( + self, + subject_template_name="registration/password_reset_subject.txt", + email_template_name="api_password_reset_email.html", + token_generator=default_token_generator, + from_email=None, + ): """ Generates a one-use only link for resetting password and sends to the user. @@ -110,8 +123,12 @@ def save(self, if not user.has_usable_password(): continue subject, email = get_password_reset_email( - user, reset_url, subject_template_name, email_template_name, - email_subject=self.email_subject) + user, + reset_url, + subject_template_name, + email_template_name, + email_subject=self.email_subject, + ) send_mail(subject, email, from_email, [user.email]) @@ -119,17 +136,17 @@ def save(self, class PasswordResetSerializer(serializers.Serializer): email = serializers.EmailField(label=_("Email"), max_length=254) reset_url = serializers.URLField(label=_("Reset URL"), max_length=254) - email_subject = serializers.CharField(label=_("Email Subject"), - required=False, max_length=78, - allow_blank=True) + email_subject = serializers.CharField( + label=_("Email Subject"), required=False, max_length=78, allow_blank=True + ) def validate_email(self, value): users = User.objects.filter(email__iexact=value) if users.count() == 0: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + raise serializers.ValidationError( + _("User '%(value)s' does not exist." % {"value": value}) + ) return value @@ -157,8 +174,8 @@ def validate_uid(self, value): return value def validate(self, attrs): - user = get_user_from_uid(attrs.get('uid')) - token = attrs.get('token') + user = get_user_from_uid(attrs.get("uid")) + token = attrs.get("token") if not default_token_generator.check_token(user, token): raise serializers.ValidationError(_("Invalid token: %s") % token) diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index 1d00d673e9..d9f3f671cd 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -2,7 +2,7 @@ """ Project Serializer module. """ -from future.utils import listvalues +from six import itervalues from django.conf import settings from django.contrib.auth.models import User @@ -13,20 +13,33 @@ from rest_framework import serializers from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import (get_organization_members_team, - get_or_create_organization_owners_team) +from onadata.apps.api.tools import ( + get_organization_members_team, + get_or_create_organization_owners_team, +) from onadata.apps.logger.models import Project, XForm from onadata.libs.permissions import ( - OwnerRole, ReadOnlyRole, ManagerRole, get_role, is_organization) -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer + OwnerRole, + ReadOnlyRole, + ManagerRole, + get_role, + is_organization, +) +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.cache_tools import ( - PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, - PROJECT_LINKED_DATAVIEWS, PROJ_OWNER_CACHE, safe_delete) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_PERM_CACHE, + PROJ_SUB_DATE_CACHE, + PROJ_TEAM_USERS_CACHE, + PROJECT_LINKED_DATAVIEWS, + PROJ_OWNER_CACHE, + safe_delete, +) from onadata.libs.utils.decorators import check_obj @@ -35,8 +48,11 @@ def get_project_xforms(project): Returns an XForm queryset from project. The prefetched `xforms_prefetch` or `xform_set.filter()` queryset. """ - return (project.xforms_prefetch if hasattr(project, 'xforms_prefetch') else - project.xform_set.filter(deleted_at__isnull=True)) + return ( + project.xforms_prefetch + if hasattr(project, "xforms_prefetch") + else project.xform_set.filter(deleted_at__isnull=True) + ) @check_obj @@ -46,20 +62,17 @@ def get_last_submission_date(project): :param project: The project to find the last submission date for. """ - last_submission_date = cache.get( - '{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk)) + last_submission_date = cache.get("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk)) if last_submission_date: return last_submission_date xforms = get_project_xforms(project) dates = [ - x.last_submission_time for x in xforms - if x.last_submission_time is not None + x.last_submission_time for x in xforms if x.last_submission_time is not None ] dates.sort(reverse=True) last_submission_date = dates[0] if dates else None - cache.set('{}{}'.format(PROJ_SUB_DATE_CACHE, project.pk), - last_submission_date) + cache.set("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk), last_submission_date) return last_submission_date @@ -70,12 +83,12 @@ def get_num_datasets(project): :param project: The project to find datasets for. """ - count = cache.get('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk)) + count = cache.get("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk)) if count: return count count = len(get_project_xforms(project)) - cache.set('{}{}'.format(PROJ_NUM_DATASET_CACHE, project.pk), count) + cache.set("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk), count) return count @@ -91,8 +104,8 @@ def get_team_permissions(team, project): Return team permissions. """ return project.projectgroupobjectpermission_set.filter( - group__pk=team.pk).values_list( - 'permission__codename', flat=True) + group__pk=team.pk + ).values_list("permission__codename", flat=True) @check_obj @@ -100,7 +113,7 @@ def get_teams(project): """ Return the teams with access to the project. """ - teams_users = cache.get('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk)) + teams_users = cache.get("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk)) if teams_users: return teams_users @@ -112,13 +125,11 @@ def get_teams(project): users = [user.username for user in team.user_set.all()] perms = get_team_permissions(team, project) - teams_users.append({ - "name": team.name, - "role": get_role(perms, project), - "users": users - }) + teams_users.append( + {"name": team.name, "role": get_role(perms, project), "users": users} + ) - cache.set('{}{}'.format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) + cache.set("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) return teams_users @@ -128,22 +139,23 @@ def get_users(project, context, all_perms=True): Return a list of users and organizations that have access to the project. """ if all_perms: - users = cache.get('{}{}'.format(PROJ_PERM_CACHE, project.pk)) + users = cache.get("{}{}".format(PROJ_PERM_CACHE, project.pk)) if users: return users data = {} - request_user = context['request'].user + request_user = context["request"].user if not request_user.is_anonymous: request_user_perms = [ perm.permission.codename for perm in project.projectuserobjectpermission_set.filter( - user=request_user)] + user=request_user + ) + ] request_user_role = get_role(request_user_perms, project) - request_user_is_admin = request_user_role in [ - OwnerRole.name, ManagerRole.name] + request_user_is_admin = request_user_role in [OwnerRole.name, ManagerRole.name] else: request_user_is_admin = False @@ -151,29 +163,31 @@ def get_users(project, context, all_perms=True): if perm.user_id not in data: user = perm.user - if all_perms or user in [ - request_user, project.organization - ] or request_user_is_admin: + if ( + all_perms + or user in [request_user, project.organization] + or request_user_is_admin + ): data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append(perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], project) - del data[k]['permissions'] + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], project) + del data[k]["permissions"] - results = listvalues(data) + results = list(itervalues(data)) if all_perms: - cache.set('{}{}'.format(PROJ_PERM_CACHE, project.pk), results) + cache.set("{}{}".format(PROJ_PERM_CACHE, project.pk), results) return results @@ -187,61 +201,77 @@ class BaseProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectXFormSerializer class. """ - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'is_merged_dataset') + fields = ("name", "formid", "id_string", "is_merged_dataset") class ProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ ProjectXFormSerializer class - to return project xform info. """ + url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') - formid = serializers.ReadOnlyField(source='id') - name = serializers.ReadOnlyField(source='title') + view_name="xform-detail", lookup_field="pk" + ) + formid = serializers.ReadOnlyField(source="id") + name = serializers.ReadOnlyField(source="title") published_by_formbuilder = serializers.SerializerMethodField() class Meta: model = XForm - fields = ('name', 'formid', 'id_string', 'num_of_submissions', - 'downloadable', 'encrypted', 'published_by_formbuilder', - 'last_submission_time', 'date_created', 'url', - 'last_updated_at', 'is_merged_dataset', ) + fields = ( + "name", + "formid", + "id_string", + "num_of_submissions", + "downloadable", + "encrypted", + "published_by_formbuilder", + "last_submission_time", + "date_created", + "url", + "last_updated_at", + "is_merged_dataset", + ) def get_published_by_formbuilder(self, obj): # pylint: disable=no-self-use """ Returns true if the form was published by formbuilder. """ - metadata = obj.metadata_set.filter( - data_type='published_by_formbuilder').first() - return (metadata and hasattr(metadata, 'data_value') - and metadata.data_value) + metadata = obj.metadata_set.filter(data_type="published_by_formbuilder").first() + return metadata and hasattr(metadata, "data_value") and metadata.data_value class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectSerializer class. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -250,25 +280,39 @@ class BaseProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project fields = [ - 'url', 'projectid', 'owner', 'created_by', 'metadata', 'starred', - 'users', 'forms', 'public', 'tags', 'num_datasets', - 'last_submission_date', 'teams', 'name', 'date_created', - 'date_modified', 'deleted_at' + "url", + "projectid", + "owner", + "created_by", + "metadata", + "starred", + "users", + "forms", + "public", + "tags", + "num_datasets", + "last_submission_date", + "teams", + "name", + "date_created", + "date_modified", + "deleted_at", ] def get_starred(self, obj): """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_users(self, obj): """ Return a list of users and organizations that have access to the project. """ - owner_query_param_in_request = 'request' in self.context and\ - "owner" in self.context['request'].GET + owner_query_param_in_request = ( + "request" in self.context and "owner" in self.context["request"].GET + ) return get_users(obj, self.context, owner_query_param_in_request) @check_obj @@ -276,16 +320,17 @@ def get_forms(self, obj): """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk)) + forms = cache.get("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk)) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = BaseProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) + cache.set("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) return forms @@ -312,11 +357,12 @@ def can_add_project_to_profile(user, organization): """ Check if user has permission to add a project to a profile. """ - perms = 'can_add_project' - if user != organization and \ - not user.has_perm(perms, organization.profile) and \ - not user.has_perm( - perms, OrganizationProfile.objects.get(user=organization)): + perms = "can_add_project" + if ( + user != organization + and not user.has_perm(perms, organization.profile) + and not user.has_perm(perms, OrganizationProfile.objects.get(user=organization)) + ): return False return True @@ -326,22 +372,27 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): """ ProjectSerializer class - creates and updates a project. """ - projectid = serializers.ReadOnlyField(source='id') + + projectid = serializers.ReadOnlyField(source="id") url = serializers.HyperlinkedIdentityField( - view_name='project-detail', lookup_field='pk') + view_name="project-detail", lookup_field="pk" + ) owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='organization', - lookup_field='username', + view_name="user-detail", + source="organization", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', lookup_field='username', read_only=True) + view_name="user-detail", lookup_field="username", read_only=True + ) metadata = JsonField(required=False) starred = serializers.SerializerMethodField() users = serializers.SerializerMethodField() forms = serializers.SerializerMethodField() - public = serializers.BooleanField(source='shared') + public = serializers.BooleanField(source="shared") tags = TagListSerializer(read_only=True) num_datasets = serializers.SerializerMethodField() last_submission_date = serializers.SerializerMethodField() @@ -350,21 +401,22 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Project - exclude = ('shared', 'user_stars', 'deleted_by', 'organization') + exclude = ("shared", "user_stars", "deleted_by", "organization") def validate(self, attrs): - name = attrs.get('name') - organization = attrs.get('organization') + name = attrs.get("name") + organization = attrs.get("organization") if not self.instance and organization: project_w_same_name = Project.objects.filter( - name__iexact=name, - organization=organization) + name__iexact=name, organization=organization + ) if project_w_same_name: - raise serializers.ValidationError({ - 'name': _(u"Project {} already exists.".format(name))}) + raise serializers.ValidationError( + {"name": _("Project {} already exists.".format(name))} + ) else: organization = organization or self.instance.organization - request = self.context['request'] + request = self.context["request"] try: has_perm = can_add_project_to_profile(request.user, organization) except OrganizationProfile.DoesNotExist: @@ -372,12 +424,15 @@ def validate(self, attrs): # A user does not require permissions to the user's account forms. has_perm = False if not has_perm: - raise serializers.ValidationError({ - 'owner': - _("You do not have permission to create a project " - "in the organization %(organization)s." % { - 'organization': organization}) - }) + raise serializers.ValidationError( + { + "owner": _( + "You do not have permission to create a project " + "in the organization %(organization)s." + % {"organization": organization} + ) + } + ) return attrs def validate_public(self, value): # pylint: disable=no-self-use @@ -386,7 +441,8 @@ def validate_public(self, value): # pylint: disable=no-self-use """ if not settings.ALLOW_PUBLIC_DATASETS and value: raise serializers.ValidationError( - _('Public projects are currently disabled.')) + _("Public projects are currently disabled.") + ) return value def validate_metadata(self, value): # pylint: disable=no-self-use @@ -404,33 +460,32 @@ def validate_metadata(self, value): # pylint: disable=no-self-use return value def update(self, instance, validated_data): - metadata = JsonField.to_json(validated_data.get('metadata')) + metadata = JsonField.to_json(validated_data.get("metadata")) if metadata is None: metadata = dict() - owner = validated_data.get('organization') + owner = validated_data.get("organization") if self.partial and metadata: if not isinstance(instance.metadata, dict): instance.metadata = {} instance.metadata.update(metadata) - validated_data['metadata'] = instance.metadata + validated_data["metadata"] = instance.metadata if self.partial and owner: # give the new owner permissions set_owners_permission(owner, instance) if is_organization(owner.profile): - owners_team = get_or_create_organization_owners_team( - owner.profile) + owners_team = get_or_create_organization_owners_team(owner.profile) members_team = get_organization_members_team(owner.profile) OwnerRole.add(owners_team, instance) ReadOnlyRole.add(members_team, instance) owners = owners_team.user_set.all() # Owners are also members members = members_team.user_set.exclude( - username__in=[ - user.username for user in owners]) + username__in=[user.username for user in owners] + ) # Exclude new owner if in members members = members.exclude(username=owner.username) @@ -439,47 +494,50 @@ def update(self, instance, validated_data): [ReadOnlyRole.add(member, instance) for member in members] # clear cache - safe_delete('{}{}'.format(PROJ_PERM_CACHE, instance.pk)) + safe_delete("{}{}".format(PROJ_PERM_CACHE, instance.pk)) - project = super(ProjectSerializer, self)\ - .update(instance, validated_data) + project = super(ProjectSerializer, self).update(instance, validated_data) - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) return instance @track_object_event( - user_field='created_by', + user_field="created_by", properties={ - 'created_by': 'created_by', - 'project_id': 'pk', - 'project_name': 'name'} + "created_by": "created_by", + "project_id": "pk", + "project_name": "name", + }, ) def create(self, validated_data): - metadata = validated_data.get('metadata', dict()) + metadata = validated_data.get("metadata", dict()) if metadata is None: metadata = dict() - created_by = self.context['request'].user + created_by = self.context["request"].user try: project = Project.objects.create( # pylint: disable=E1101 - name=validated_data.get('name'), - organization=validated_data.get('organization'), + name=validated_data.get("name"), + organization=validated_data.get("organization"), created_by=created_by, - shared=validated_data.get('shared', False), - metadata=metadata) + shared=validated_data.get("shared", False), + metadata=metadata, + ) except IntegrityError: raise serializers.ValidationError( - "The fields name, organization must make a unique set.") + "The fields name, organization must make a unique set." + ) else: - project.xform_set.exclude(shared=project.shared)\ - .update(shared=project.shared, shared_data=project.shared) - request = self.context.get('request') - serializer = ProjectSerializer( - project, context={'request': request}) + project.xform_set.exclude(shared=project.shared).update( + shared=project.shared, shared_data=project.shared + ) + request = self.context.get("request") + serializer = ProjectSerializer(project, context={"request": request}) response = serializer.data - cache.set(f'{PROJ_OWNER_CACHE}{project.pk}', response) + cache.set(f"{PROJ_OWNER_CACHE}{project.pk}", response) return project def get_users(self, obj): # pylint: disable=no-self-use @@ -494,15 +552,16 @@ def get_forms(self, obj): # pylint: disable=no-self-use """ Return list of xforms in the project. """ - forms = cache.get('{}{}'.format(PROJ_FORMS_CACHE, obj.pk)) + forms = cache.get("{}{}".format(PROJ_FORMS_CACHE, obj.pk)) if forms: return forms xforms = get_project_xforms(obj) - request = self.context.get('request') + request = self.context.get("request") serializer = ProjectXFormSerializer( - xforms, context={'request': request}, many=True) + xforms, context={"request": request}, many=True + ) forms = list(serializer.data) - cache.set('{}{}'.format(PROJ_FORMS_CACHE, obj.pk), forms) + cache.set("{}{}".format(PROJ_FORMS_CACHE, obj.pk), forms) return forms @@ -522,7 +581,7 @@ def get_starred(self, obj): # pylint: disable=no-self-use """ Return True if request user has starred this project. """ - return is_starred(obj, self.context['request']) + return is_starred(obj, self.context["request"]) def get_teams(self, obj): # pylint: disable=no-self-use """ @@ -535,18 +594,21 @@ def get_data_views(self, obj): """ Return a list of filtered datasets. """ - data_views = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk)) + data_views = cache.get("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk)) if data_views: return data_views - data_views_obj = obj.dataview_prefetch if \ - hasattr(obj, 'dataview_prefetch') else\ - obj.dataview_set.filter(deleted_at__isnull=True) + data_views_obj = ( + obj.dataview_prefetch + if hasattr(obj, "dataview_prefetch") + else obj.dataview_set.filter(deleted_at__isnull=True) + ) serializer = DataViewMinimalSerializer( - data_views_obj, many=True, context=self.context) + data_views_obj, many=True, context=self.context + ) data_views = list(serializer.data) - cache.set('{}{}'.format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) + cache.set("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) return data_views diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index e93fb24ce9..5e99fa0ffa 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -2,7 +2,7 @@ from django.http import Http404 from django.urls import resolve, get_script_prefix, Resolver404 from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from guardian.shortcuts import get_users_with_perms from rest_framework import serializers @@ -19,32 +19,32 @@ class GenericRelatedField(serializers.HyperlinkedRelatedField): default_error_messages = { - 'incorrect_match': _('`{input}` is not a valid relation.') + "incorrect_match": _("`{input}` is not a valid relation.") } def __init__(self, *args, **kwargs): - self.view_names = ['xform-detail', 'dataviews-detail'] + self.view_names = ["xform-detail", "dataviews-detail"] self.resolve = resolve self.reverse = reverse - self.format = kwargs.pop('format', 'json') + self.format = kwargs.pop("format", "json") super(serializers.RelatedField, self).__init__(*args, **kwargs) def _setup_field(self, view_name): self.lookup_url_kwarg = self.lookup_field - if view_name == 'xform-detail': + if view_name == "xform-detail": self.queryset = XForm.objects.all() - if view_name == 'dataviews-detail': + if view_name == "dataviews-detail": self.queryset = DataView.objects.all() def to_representation(self, value): if isinstance(value, XForm): - self.view_name = 'xform-detail' + self.view_name = "xform-detail" elif isinstance(value, DataView): - self.view_name = 'dataviews-detail' + self.view_name = "dataviews-detail" else: - raise Exception(_(u"Unknown type for content_object.")) + raise Exception(_("Unknown type for content_object.")) self._setup_field(self.view_name) @@ -52,31 +52,31 @@ def to_representation(self, value): def to_internal_value(self, data): try: - http_prefix = data.startswith(('http:', 'https:')) + http_prefix = data.startswith(("http:", "https:")) except AttributeError: - self.fail('incorrect_type', data_type=type(data).__name__) + self.fail("incorrect_type", data_type=type(data).__name__) input_data = data if http_prefix: # If needed convert absolute URLs to relative path data = urlparse(data).path prefix = get_script_prefix() if data.startswith(prefix): - data = '/' + data[len(prefix):] + data = "/" + data[len(prefix) :] try: match = self.resolve(data) except Resolver404: - self.fail('no_match') + self.fail("no_match") if match.view_name not in self.view_names: - self.fail('incorrect_match', input=input_data) + self.fail("incorrect_match", input=input_data) self._setup_field(match.view_name) try: return self.get_object(match.view_name, match.args, match.kwargs) except (ObjectDoesNotExist, TypeError, ValueError): - self.fail('does_not_exist') + self.fail("does_not_exist") return data @@ -84,8 +84,7 @@ def to_internal_value(self, data): class WidgetSerializer(serializers.HyperlinkedModelSerializer): id = serializers.ReadOnlyField() url = serializers.HyperlinkedIdentityField( - view_name='widgets-detail', - lookup_field='pk' + view_name="widgets-detail", lookup_field="pk" ) content_object = GenericRelatedField() key = serializers.CharField(read_only=True) @@ -95,17 +94,30 @@ class WidgetSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Widget - fields = ('id', 'url', 'key', 'title', 'description', 'widget_type', - 'order', 'view_type', 'column', 'group_by', 'content_object', - 'data', 'aggregation', 'metadata') + fields = ( + "id", + "url", + "key", + "title", + "description", + "widget_type", + "order", + "view_type", + "column", + "group_by", + "content_object", + "data", + "aggregation", + "metadata", + ) def get_data(self, obj): # Get the request obj - request = self.context.get('request') + request = self.context.get("request") # Check if data flag is present - data_flag = request.GET.get('data') - key = request.GET.get('key') + data_flag = request.GET.get("data") + key = request.GET.get("key") if (str2bool(data_flag) or key) and obj: data = Widget.query_data(obj) @@ -115,11 +127,11 @@ def get_data(self, obj): return data def validate(self, attrs): - column = attrs.get('column') + column = attrs.get("column") # Get the form - if 'content_object' in attrs: - content_object = attrs.get('content_object') + if "content_object" in attrs: + content_object = attrs.get("content_object") if isinstance(content_object, XForm): xform = content_object @@ -131,11 +143,11 @@ def validate(self, attrs): # Check if column exists in xform get_field_from_field_xpath(column, xform) except Http404: - raise serializers.ValidationError({ - 'column': (u"'{}' not in the form.".format(column)) - }) + raise serializers.ValidationError( + {"column": ("'{}' not in the form.".format(column))} + ) - order = attrs.get('order') + order = attrs.get("order") # Set the order if order: @@ -144,19 +156,20 @@ def validate(self, attrs): return attrs def validate_content_object(self, value): - request = self.context.get('request') + request = self.context.get("request") users = get_users_with_perms( value.project, attach_perms=False, with_group_users=False ) profile = value.project.organization.profile # Shared or an admin in the organization - if request.user not in users and not\ - is_organization(profile) and not\ - OwnerRole.user_has_role(request.user, - profile): - raise serializers.ValidationError(_( - u"You don't have permission to the Project." - )) + if ( + request.user not in users + and not is_organization(profile) + and not OwnerRole.user_has_role(request.user, profile) + ): + raise serializers.ValidationError( + _("You don't have permission to the Project.") + ) return value diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index fcf5408172..929e10011f 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -11,8 +11,8 @@ from django.core.validators import URLValidator from django.db.models import Count from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import listvalues +from six.moves.urllib.parse import urlparse +from six import itervalues from requests.exceptions import ConnectionError from rest_framework import serializers from rest_framework.reverse import reverse @@ -22,24 +22,27 @@ from onadata.apps.logger.models import XFormVersion from onadata.libs.exceptions import EnketoError from onadata.libs.permissions import get_role, is_organization -from onadata.libs.serializers.dataview_serializer import \ - DataViewMinimalSerializer +from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.metadata_serializer import MetaDataSerializer from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.cache_tools import ( - ENKETO_PREVIEW_URL_CACHE, ENKETO_URL_CACHE, ENKETO_SINGLE_SUBMIT_URL_CACHE, - XFORM_LINKED_DATAVIEWS, XFORM_METADATA_CACHE, XFORM_PERMISSIONS_CACHE, - XFORM_DATA_VERSIONS, XFORM_COUNT) -from onadata.libs.utils.common_tags import (GROUP_DELIMETER_TAG, - REPEAT_INDEX_TAGS) + ENKETO_PREVIEW_URL_CACHE, + ENKETO_URL_CACHE, + ENKETO_SINGLE_SUBMIT_URL_CACHE, + XFORM_LINKED_DATAVIEWS, + XFORM_METADATA_CACHE, + XFORM_PERMISSIONS_CACHE, + XFORM_DATA_VERSIONS, + XFORM_COUNT, +) +from onadata.libs.utils.common_tags import GROUP_DELIMETER_TAG, REPEAT_INDEX_TAGS from onadata.libs.utils.decorators import check_obj -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) def _create_enketo_urls(request, xform): @@ -50,9 +53,13 @@ def _create_enketo_urls(request, xform): :param xform: :return: enketo urls """ - form_url = get_form_url(request, xform.user.username, - settings.ENKETO_PROTOCOL, xform_pk=xform.pk, - generate_consistent_urls=True) + form_url = get_form_url( + request, + xform.user.username, + settings.ENKETO_PROTOCOL, + xform_pk=xform.pk, + generate_consistent_urls=True, + ) data = {} try: enketo_urls = get_enketo_urls(form_url, xform.id_string) @@ -60,16 +67,15 @@ def _create_enketo_urls(request, xform): return data offline_url = enketo_urls.get("offline_url") MetaData.enketo_url(xform, offline_url) - data['offline_url'] = offline_url - if 'preview_url' in enketo_urls: + data["offline_url"] = offline_url + if "preview_url" in enketo_urls: preview_url = enketo_urls.get("preview_url") MetaData.enketo_preview_url(xform, preview_url) - data['preview_url'] = preview_url - if 'single_url' in enketo_urls: + data["preview_url"] = preview_url + if "single_url" in enketo_urls: single_url = enketo_urls.get("single_url") - MetaData.enketo_single_submit_url( - xform, single_url) - data['single_url'] = single_url + MetaData.enketo_single_submit_url(xform, single_url) + data["single_url"] = single_url except ConnectionError as e: logging.exception("Connection Error: %s" % e) except EnketoError as e: @@ -87,22 +93,26 @@ def _set_cache(cache_key, cache_data, obj): :param obj: :return: Data that has been cached """ - cache.set('{}{}'.format(cache_key, obj.pk), cache_data) + cache.set("{}{}".format(cache_key, obj.pk), cache_data) return cache_data def user_to_username(item): - item['user'] = item['user'].username + item["user"] = item["user"].username return item def clean_public_key(value): - if value.startswith('-----BEGIN PUBLIC KEY-----') and\ - value.endswith('-----END PUBLIC KEY-----'): - return value.replace('-----BEGIN PUBLIC KEY-----', - '').replace('-----END PUBLIC KEY-----', - '').replace(' ', '').rstrip() + if value.startswith("-----BEGIN PUBLIC KEY-----") and value.endswith( + "-----END PUBLIC KEY-----" + ): + return ( + value.replace("-----BEGIN PUBLIC KEY-----", "") + .replace("-----END PUBLIC KEY-----", "") + .replace(" ", "") + .rstrip() + ) class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): @@ -111,27 +121,27 @@ class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): Credits: https://stackoverflow.com/a/31161585 """ - lookup_fields = (('pk', 'pk'),) + + lookup_fields = (("pk", "pk"),) def __init__(self, *args, **kwargs): - self.lookup_fields = kwargs.pop('lookup_fields', self.lookup_fields) + self.lookup_fields = kwargs.pop("lookup_fields", self.lookup_fields) super(MultiLookupIdentityField, self).__init__(*args, **kwargs) def get_url(self, obj, view_name, request, format): kwargs = {} for model_field, url_param in self.lookup_fields: attr = obj - for field in model_field.split('__'): + for field in model_field.split("__"): attr = getattr(attr, field) kwargs[url_param] = attr - if not format and hasattr(self, 'format'): + if not format and hasattr(self, "format"): fmt = self.format else: fmt = format - return reverse( - view_name, kwargs=kwargs, request=request, format=fmt) + return reverse(view_name, kwargs=kwargs, request=request, format=fmt) class XFormMixin(object): @@ -155,52 +165,48 @@ def _get_metadata(self, obj, key): def get_users(self, obj): xform_perms = [] if obj: - xform_perms = cache.get( - '{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk)) + xform_perms = cache.get("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk)) if xform_perms: return xform_perms - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), - xform_perms) + cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) data = {} for perm in obj.xformuserobjectpermission_set.all(): if perm.user_id not in data: user = perm.user data[perm.user_id] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'metadata': user.profile.metadata, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'user': user.username + "permissions": [], + "is_org": is_organization(user.profile), + "metadata": user.profile.metadata, + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.username, } if perm.user_id in data: - data[perm.user_id]['permissions'].append( - perm.permission.codename) + data[perm.user_id]["permissions"].append(perm.permission.codename) for k in list(data): - data[k]['permissions'].sort() - data[k]['role'] = get_role(data[k]['permissions'], XForm) - del (data[k]['permissions']) + data[k]["permissions"].sort() + data[k]["role"] = get_role(data[k]["permissions"], XForm) + del data[k]["permissions"] - xform_perms = listvalues(data) + xform_perms = list(itervalues(data)) - cache.set('{}{}'.format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) + cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) return xform_perms def get_enketo_url(self, obj): if obj: - _enketo_url = cache.get('{}{}'.format(ENKETO_URL_CACHE, obj.pk)) + _enketo_url = cache.get("{}{}".format(ENKETO_URL_CACHE, obj.pk)) if _enketo_url: return _enketo_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") return _set_cache(ENKETO_URL_CACHE, url, obj) @@ -209,35 +215,33 @@ def get_enketo_url(self, obj): def get_enketo_single_submit_url(self, obj): if obj: _enketo_single_submit_url = cache.get( - '{}{}'.format(ENKETO_SINGLE_SUBMIT_URL_CACHE, - obj.pk)) + "{}{}".format(ENKETO_SINGLE_SUBMIT_URL_CACHE, obj.pk) + ) if _enketo_single_submit_url: return _enketo_single_submit_url - url = self._get_metadata(obj, 'enketo_url') + url = self._get_metadata(obj, "enketo_url") if url is None: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('offline_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("offline_url") - return _set_cache( - ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) + return _set_cache(ENKETO_SINGLE_SUBMIT_URL_CACHE, url, obj) return None def get_enketo_preview_url(self, obj): if obj: _enketo_preview_url = cache.get( - '{}{}'.format(ENKETO_PREVIEW_URL_CACHE, obj.pk)) + "{}{}".format(ENKETO_PREVIEW_URL_CACHE, obj.pk) + ) if _enketo_preview_url: return _enketo_preview_url - url = self._get_metadata(obj, 'enketo_preview_url') + url = self._get_metadata(obj, "enketo_preview_url") if url is None: try: - enketo_urls = _create_enketo_urls( - self.context.get('request'), obj) - url = enketo_urls.get('preview_url') + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = enketo_urls.get("preview_url") except Exception: return url @@ -247,14 +251,16 @@ def get_enketo_preview_url(self, obj): def get_data_views(self, obj): if obj: - key = '{}{}'.format(XFORM_LINKED_DATAVIEWS, obj.pk) + key = "{}{}".format(XFORM_LINKED_DATAVIEWS, obj.pk) data_views = cache.get(key) if data_views: return data_views data_views = DataViewMinimalSerializer( obj.dataview_set.filter(deleted_at__isnull=True), - many=True, context=self.context).data + many=True, + context=self.context, + ).data cache.set(key, list(data_views)) @@ -263,7 +269,7 @@ def get_data_views(self, obj): def get_num_of_submissions(self, obj): if obj: - key = '{}{}'.format(XFORM_COUNT, obj.pk) + key = "{}{}".format(XFORM_COUNT, obj.pk) count = cache.get(key) if count: return count @@ -280,43 +286,49 @@ def get_last_submission_time(self, obj): If a form is a merged dataset then it is picked from the list of forms attached to that merged dataset. """ - if 'last_submission_time' not in self.fields: + if "last_submission_time" not in self.fields: return None if obj.is_merged_dataset: values = [ x.last_submission_time.isoformat() - for x in obj.mergedxform.xforms.only('last_submission_time') + for x in obj.mergedxform.xforms.only("last_submission_time") if x.last_submission_time ] if values: return sorted(values, reverse=True)[0] - return obj.last_submission_time.isoformat() \ - if obj.last_submission_time else None + return ( + obj.last_submission_time.isoformat() if obj.last_submission_time else None + ) class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + formid = serializers.ReadOnlyField(source="id") owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -328,37 +340,58 @@ class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): - formid = serializers.ReadOnlyField(source='id') + formid = serializers.ReadOnlyField(source="id") metadata = serializers.SerializerMethodField() owner = serializers.HyperlinkedRelatedField( - view_name='user-detail', - source='user', - lookup_field='username', + view_name="user-detail", + source="user", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)) - public = serializers.BooleanField(source='shared') - public_data = serializers.BooleanField(source='shared_data') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), + ) + public = serializers.BooleanField(source="shared") + public_data = serializers.BooleanField(source="shared_data") public_key = serializers.CharField(required=False) require_auth = serializers.BooleanField() submission_count_for_today = serializers.ReadOnlyField() tags = TagListSerializer(read_only=True) title = serializers.CharField(max_length=255) url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') + view_name="xform-detail", lookup_field="pk" + ) users = serializers.SerializerMethodField() enketo_url = serializers.SerializerMethodField() enketo_preview_url = serializers.SerializerMethodField() @@ -371,27 +404,42 @@ class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): class Meta: model = XForm - read_only_fields = ('json', 'xml', 'date_created', 'date_modified', - 'encrypted', 'bamboo_dataset', - 'last_submission_time', 'is_merged_dataset', - 'xls_available') - exclude = ('json', 'xml', 'xls', 'user', 'has_start_time', 'shared', - 'shared_data', 'deleted_at', 'deleted_by') + read_only_fields = ( + "json", + "xml", + "date_created", + "date_modified", + "encrypted", + "bamboo_dataset", + "last_submission_time", + "is_merged_dataset", + "xls_available", + ) + exclude = ( + "json", + "xml", + "xls", + "user", + "has_start_time", + "shared", + "shared_data", + "deleted_at", + "deleted_by", + ) def get_metadata(self, obj): xform_metadata = [] if obj: - xform_metadata = cache.get( - '{}{}'.format(XFORM_METADATA_CACHE, obj.pk)) + xform_metadata = cache.get("{}{}".format(XFORM_METADATA_CACHE, obj.pk)) if xform_metadata: return xform_metadata xform_metadata = list( MetaDataSerializer( - obj.metadata_set.all(), many=True, context=self.context) - .data) - cache.set('{}{}'.format(XFORM_METADATA_CACHE, obj.pk), - xform_metadata) + obj.metadata_set.all(), many=True, context=self.context + ).data + ) + cache.set("{}{}".format(XFORM_METADATA_CACHE, obj.pk), xform_metadata) return xform_metadata @@ -402,11 +450,11 @@ def validate_public_key(self, value): # pylint: disable=no-self-use package """ try: - load_pem_public_key( - value.encode('utf-8'), backend=default_backend()) + load_pem_public_key(value.encode("utf-8"), backend=default_backend()) except ValueError: raise serializers.ValidationError( - _('The public key is not a valid base64 RSA key')) + _("The public key is not a valid base64 RSA key") + ) return clean_public_key(value) def _check_if_allowed_public(self, value): # pylint: disable=no-self-use @@ -415,8 +463,7 @@ def _check_if_allowed_public(self, value): # pylint: disable=no-self-use forms """ if not settings.ALLOW_PUBLIC_DATASETS and value: - raise serializers.ValidationError( - _('Public forms are currently disabled.')) + raise serializers.ValidationError(_("Public forms are currently disabled.")) return value def validate_public_data(self, value): @@ -434,7 +481,7 @@ def validate_public(self, value): def get_form_versions(self, obj): versions = [] if obj: - versions = cache.get('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk)) + versions = cache.get("{}{}".format(XFORM_DATA_VERSIONS, obj.pk)) if versions: return versions @@ -443,11 +490,12 @@ def get_form_versions(self, obj): versions = list( Instance.objects.filter(xform=obj, deleted_at__isnull=True) - .values('version').annotate(total=Count('version'))) + .values("version") + .annotate(total=Count("version")) + ) if versions: - cache.set('{}{}'.format(XFORM_DATA_VERSIONS, obj.pk), - list(versions)) + cache.set("{}{}".format(XFORM_DATA_VERSIONS, obj.pk), list(versions)) return versions @@ -460,58 +508,59 @@ def get_has_id_string_changed(self, obj): class XFormListSerializer(serializers.Serializer): - formID = serializers.ReadOnlyField(source='id_string') - name = serializers.ReadOnlyField(source='title') + formID = serializers.ReadOnlyField(source="id_string") + name = serializers.ReadOnlyField(source="title") version = serializers.ReadOnlyField() hash = serializers.ReadOnlyField() - descriptionText = serializers.ReadOnlyField(source='description') - downloadUrl = serializers.SerializerMethodField('get_url') - manifestUrl = serializers.SerializerMethodField('get_manifest_url') + descriptionText = serializers.ReadOnlyField(source="description") + downloadUrl = serializers.SerializerMethodField("get_url") + manifestUrl = serializers.SerializerMethodField("get_manifest_url") @check_obj def get_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") - return reverse('download_xform', kwargs=kwargs, request=request) + return reverse("download_xform", kwargs=kwargs, request=request) @check_obj def get_manifest_url(self, obj): - kwargs = {'pk': obj.pk, 'username': obj.user.username} - request = self.context.get('request') - object_list = MetaData.objects.filter(data_type='media', - object_id=obj.pk) + kwargs = {"pk": obj.pk, "username": obj.user.username} + request = self.context.get("request") + object_list = MetaData.objects.filter(data_type="media", object_id=obj.pk) if object_list: - return reverse('manifest-url', kwargs=kwargs, request=request) + return reverse("manifest-url", kwargs=kwargs, request=request) return None class XFormManifestSerializer(serializers.Serializer): filename = serializers.SerializerMethodField() hash = serializers.SerializerMethodField() - downloadUrl = serializers.SerializerMethodField('get_url') + downloadUrl = serializers.SerializerMethodField("get_url") @check_obj def get_url(self, obj): kwargs = { - 'pk': obj.content_object.pk, - 'username': obj.content_object.user.username, - 'metadata': obj.pk + "pk": obj.content_object.pk, + "username": obj.content_object.user.username, + "metadata": obj.pk, } - request = self.context.get('request') + request = self.context.get("request") try: - fmt = obj.data_value[obj.data_value.rindex('.') + 1:] + fmt = obj.data_value[obj.data_value.rindex(".") + 1 :] except ValueError: - fmt = 'csv' - url = reverse( - 'xform-media', kwargs=kwargs, request=request, format=fmt.lower()) + fmt = "csv" + url = reverse("xform-media", kwargs=kwargs, request=request, format=fmt.lower()) group_delimiter = self.context.get(GROUP_DELIMETER_TAG) repeat_index_tags = self.context.get(REPEAT_INDEX_TAGS) - if group_delimiter and repeat_index_tags and fmt == 'csv': - return (url+"?%s=%s&%s=%s" % ( - GROUP_DELIMETER_TAG, group_delimiter, REPEAT_INDEX_TAGS, - repeat_index_tags)) + if group_delimiter and repeat_index_tags and fmt == "csv": + return url + "?%s=%s&%s=%s" % ( + GROUP_DELIMETER_TAG, + group_delimiter, + REPEAT_INDEX_TAGS, + repeat_index_tags, + ) return url @@ -519,38 +568,42 @@ def get_url(self, obj): def get_hash(self, obj): filename = obj.data_value hsh = obj.file_hash - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", xform pk is the # second item # other file uploads other than linked datasets have a data_file - if len(parts) > 2 and obj.data_file == '': + if len(parts) > 2 and obj.data_file == "": dataset_type = parts[0] pk = parts[1] xform = None - if dataset_type == 'xform': - xform = XForm.objects.filter(pk=pk)\ - .only('last_submission_time').first() + if dataset_type == "xform": + xform = XForm.objects.filter(pk=pk).only("last_submission_time").first() else: - data_view = DataView.objects.filter(pk=pk)\ - .only('xform__last_submission_time').first() + data_view = ( + DataView.objects.filter(pk=pk) + .only("xform__last_submission_time") + .first() + ) if data_view: xform = data_view.xform if xform and xform.last_submission_time: - hsh = u'md5:%s' % (md5( - xform.last_submission_time.isoformat().encode( - 'utf-8')).hexdigest()) + hsh = "md5:%s" % ( + md5( + xform.last_submission_time.isoformat().encode("utf-8") + ).hexdigest() + ) - return u"%s" % (hsh or 'md5:') + return "%s" % (hsh or "md5:") @check_obj def get_filename(self, obj): filename = obj.data_value - parts = filename.split(' ') + parts = filename.split(" ") # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = u'%s.csv' % parts[2] + filename = "%s.csv" % parts[2] else: try: URLValidator()(filename) @@ -565,25 +618,27 @@ def get_filename(self, obj): class XFormVersionListSerializer(serializers.ModelSerializer): xform = serializers.HyperlinkedRelatedField( - view_name='xform-detail', lookup_field='pk', - queryset=XForm.objects.filter(deleted_at__isnull=True) + view_name="xform-detail", + lookup_field="pk", + queryset=XForm.objects.filter(deleted_at__isnull=True), ) url = MultiLookupIdentityField( - view_name='form-version-detail', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) xml = MultiLookupIdentityField( - view_name='form-version-detail', - format='xml', - lookup_fields=(('xform__pk', 'pk'), ('version', 'version_id')) + view_name="form-version-detail", + format="xml", + lookup_fields=(("xform__pk", "pk"), ("version", "version_id")), ) created_by = serializers.HyperlinkedRelatedField( - view_name='user-detail', - lookup_field='username', + view_name="user-detail", + lookup_field="username", queryset=User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ), ) class Meta: model = XFormVersion - exclude = ('json', 'xls', 'id') + exclude = ("json", "xls", "id") diff --git a/onadata/libs/tests/utils/test_email.py b/onadata/libs/tests/utils/test_email.py index d1f643c512..5236df217d 100644 --- a/onadata/libs/tests/utils/test_email.py +++ b/onadata/libs/tests/utils/test_email.py @@ -1,77 +1,83 @@ -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.test import RequestFactory from django.test.utils import override_settings from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.email import ( - get_verification_email_data, get_verification_url -) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url VERIFICATION_URL = "http://ab.cd.ef" class TestEmail(TestBase): - def setUp(self): self.email = "john@doe.com" - self.username = "johndoe", + self.username = ("johndoe",) self.verification_key = "123abc" self.redirect_url = "http://red.ir.ect" - self.custom_request = RequestFactory().get( - '/path', data={'name': u'test'} - ) + self.custom_request = RequestFactory().get("/path", data={"name": "test"}) @override_settings(VERIFICATION_URL=None) def test_get_verification_url(self): # without redirect_url - verification_url = get_verification_url(**{ - "redirect_url": None, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": None, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), ) # with redirect_url - verification_url = get_verification_url(**{ - "redirect_url": self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) + verification_url = get_verification_url( + **{ + "redirect_url": self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) - string_query_params = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) + string_query_params = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } + ) self.assertEqual( verification_url, - ('http://testserver/api/v1/profiles/verify_email?%s' - % string_query_params) + ("http://testserver/api/v1/profiles/verify_email?%s" % string_query_params), ) def _get_email_data(self, include_redirect_url=False): - verification_url = get_verification_url(**{ - "redirect_url": include_redirect_url and self.redirect_url, - "request": self.custom_request, - "verification_key": self.verification_key - }) - - email_data = get_verification_email_data(**{ - "email": self.email, - "username": self.username, - "verification_url": verification_url, - "request": self.custom_request - }) - - self.assertIn('email', email_data) - self.assertIn(self.email, email_data.get('email')) - self.assertIn('subject', email_data) - self.assertIn('message_txt', email_data) + verification_url = get_verification_url( + **{ + "redirect_url": include_redirect_url and self.redirect_url, + "request": self.custom_request, + "verification_key": self.verification_key, + } + ) + + email_data = get_verification_email_data( + **{ + "email": self.email, + "username": self.username, + "verification_url": verification_url, + "request": self.custom_request, + } + ) + + self.assertIn("email", email_data) + self.assertIn(self.email, email_data.get("email")) + self.assertIn("subject", email_data) + self.assertIn("message_txt", email_data) return email_data @@ -79,33 +85,32 @@ def _get_email_data(self, include_redirect_url=False): def test_get_verification_email_data_without_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - ('http://testserver/api/v1/profiles/verify_email?' - 'verification_key=%s' % self.verification_key), - email_data.get('message_txt') + ( + "http://testserver/api/v1/profiles/verify_email?" + "verification_key=%s" % self.verification_key + ), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) def test_get_verification_email_data_with_verification_url_set(self): email_data = self._get_email_data() self.assertIn( - '{}?verification_key={}'.format( - VERIFICATION_URL, self.verification_key - ), - email_data.get('message_txt') + "{}?verification_key={}".format(VERIFICATION_URL, self.verification_key), + email_data.get("message_txt"), ) @override_settings(VERIFICATION_URL=VERIFICATION_URL) - def test_get_verification_email_data_with_verification_and_redirect_urls( - self): + def test_get_verification_email_data_with_verification_and_redirect_urls(self): email_data = self._get_email_data(include_redirect_url=True) - encoded_url = urlencode({ - 'verification_key': self.verification_key, - 'redirect_url': self.redirect_url - }) - self.assertIn( - encoded_url.replace('&', '&'), email_data.get('message_txt') + encoded_url = urlencode( + { + "verification_key": self.verification_key, + "redirect_url": self.redirect_url, + } ) + self.assertIn(encoded_url.replace("&", "&"), email_data.get("message_txt")) def test_email_data_does_not_contain_newline_chars(self): email_data = self._get_email_data(include_redirect_url=True) - self.assertNotIn('\n', email_data.get('subject')) + self.assertNotIn("\n", email_data.get("subject")) diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index f3f30e576e..ee955369c1 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -4,7 +4,7 @@ from io import StringIO from xml.parsers.expat import ExpatError -from future.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.core.files.base import ContentFile from django.core.files.storage import default_storage @@ -17,8 +17,7 @@ from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.libs.utils.common_tools import retry -from onadata.libs.utils.logger_tools import (PublishXForm, create_instance, - publish_form) +from onadata.libs.utils.logger_tools import PublishXForm, create_instance, publish_form NUM_RETRIES = 3 @@ -30,7 +29,7 @@ def django_file(file_obj, field_name, content_type): name=file_obj.name, content_type=content_type, size=file_obj.size, - charset=None + charset=None, ) @@ -49,12 +48,12 @@ def _get_form_list(xml_text): forms = [] for childNode in xml_doc.childNodes: - if childNode.nodeName == 'xforms': + if childNode.nodeName == "xforms": for xformNode in childNode.childNodes: - if xformNode.nodeName == 'xform': - id_string = node_value(xformNode, 'formID') - download_url = node_value(xformNode, 'downloadUrl') - manifest_url = node_value(xformNode, 'manifestUrl') + if xformNode.nodeName == "xform": + id_string = node_value(xformNode, "formID") + download_url = node_value(xformNode, "downloadUrl") + manifest_url = node_value(xformNode, "manifestUrl") forms.append((id_string, download_url, manifest_url)) return forms @@ -64,8 +63,8 @@ def _get_instances_uuids(xml_doc): uuids = [] for child_node in xml_doc.childNodes: - if child_node.nodeName == 'idChunk': - for id_node in child_node.getElementsByTagName('id'): + if child_node.nodeName == "idChunk": + for id_node in child_node.getElementsByTagName("id"): if id_node.childNodes: uuid = id_node.childNodes[0].nodeValue uuids.append(uuid) @@ -78,14 +77,12 @@ def __init__(self, url, username, password, user): self.url = url self.user = user self.auth = HTTPDigestAuth(username, password) - self.form_list_url = urljoin(self.url, 'formList') - self.submission_list_url = urljoin(self.url, 'view/submissionList') - self.download_submission_url = urljoin(self.url, - 'view/downloadSubmission') - self.forms_path = os.path.join( - self.user.username, 'briefcase', 'forms') + self.form_list_url = urljoin(self.url, "formList") + self.submission_list_url = urljoin(self.url, "view/submissionList") + self.download_submission_url = urljoin(self.url, "view/downloadSubmission") + self.forms_path = os.path.join(self.user.username, "briefcase", "forms") self.resumption_cursor = 0 - self.logger = logging.getLogger('console_logger') + self.logger = logging.getLogger("console_logger") def download_manifest(self, manifest_url, id_string): if self._get_response(manifest_url): @@ -96,8 +93,7 @@ def download_manifest(self, manifest_url, id_string): except ExpatError: return - manifest_path = os.path.join( - self.forms_path, id_string, 'form-media') + manifest_path = os.path.join(self.forms_path, id_string, "form-media") self.logger.debug("Downloading media files for %s" % id_string) self.download_media_files(manifest_doc, manifest_path) @@ -105,8 +101,11 @@ def download_manifest(self, manifest_url, id_string): def download_xforms(self, include_instances=False): # fetch formList if not self._get_response(self.form_list_url): - response = self._current_response.content \ - if self._current_response else "Unknown Error" + response = ( + self._current_response.content + if self._current_response + else "Unknown Error" + ) self.logger.error("Failed to download xforms %s." % response) return @@ -114,16 +113,14 @@ def download_xforms(self, include_instances=False): response = self._current_response forms = _get_form_list(response.content) - self.logger.debug('Successfull fetched %s.' % self.form_list_url) + self.logger.debug("Successfull fetched %s." % self.form_list_url) for id_string, download_url, manifest_url in forms: - form_path = os.path.join( - self.forms_path, id_string, '%s.xml' % id_string) + form_path = os.path.join(self.forms_path, id_string, "%s.xml" % id_string) if not default_storage.exists(form_path): if not self._get_response(download_url): - self.logger.error("Failed to download xform %s." - % download_url) + self.logger.error("Failed to download xform %s." % download_url) continue form_res = self._current_response @@ -140,8 +137,7 @@ def download_xforms(self, include_instances=False): if include_instances: self.download_instances(id_string) - self.logger.debug("Done downloading submissions for %s" % - id_string) + self.logger.debug("Done downloading submissions for %s" % id_string) @retry(NUM_RETRIES) def _get_response(self, url, params=None): @@ -159,7 +155,7 @@ def _get_media_response(self, url): # S3 redirects, avoid using formhub digest on S3 if head_response.status_code == 302: - url = head_response.headers.get('location') + url = head_response.headers.get("location") response = requests.get(url) success = response.status_code == 200 @@ -168,9 +164,9 @@ def _get_media_response(self, url): return success def download_media_files(self, xml_doc, media_path): - for media_node in xml_doc.getElementsByTagName('mediaFile'): - filename_node = media_node.getElementsByTagName('filename') - url_node = media_node.getElementsByTagName('downloadUrl') + for media_node in xml_doc.getElementsByTagName("mediaFile"): + filename_node = media_node.getElementsByTagName("filename") + url_node = media_node.getElementsByTagName("downloadUrl") if filename_node and url_node: filename = filename_node[0].childNodes[0].nodeValue path = os.path.join(media_path, filename) @@ -187,37 +183,41 @@ def download_media_files(self, xml_doc, media_path): def download_instances(self, form_id, cursor=0, num_entries=100): self.logger.debug("Starting submissions download for %s" % form_id) - if not self._get_response(self.submission_list_url, - params={'formId': form_id, - 'numEntries': num_entries, - 'cursor': cursor}): - self.logger.error("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + if not self._get_response( + self.submission_list_url, + params={"formId": form_id, "numEntries": num_entries, "cursor": cursor}, + ): + self.logger.error( + "Fetching %s formId: %s, cursor: %s" + % (self.submission_list_url, form_id, cursor) + ) return response = self._current_response - self.logger.debug("Fetching %s formId: %s, cursor: %s" % - (self.submission_list_url, form_id, cursor)) + self.logger.debug( + "Fetching %s formId: %s, cursor: %s" + % (self.submission_list_url, form_id, cursor) + ) try: xml_doc = clean_and_parse_xml(response.content) except ExpatError: return instances = _get_instances_uuids(xml_doc) - path = os.path.join(self.forms_path, form_id, 'instances') + path = os.path.join(self.forms_path, form_id, "instances") for uuid in instances: self.logger.debug("Fetching %s %s submission" % (uuid, form_id)) - form_str = u'%(formId)s[@version=null and @uiVersion=null]/'\ - u'%(formId)s[@key=%(instanceId)s]' % { - 'formId': form_id, - 'instanceId': uuid - } - instance_path = os.path.join(path, uuid.replace(':', ''), - 'submission.xml') + form_str = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=%(instanceId)s]" + % {"formId": form_id, "instanceId": uuid} + ) + instance_path = os.path.join(path, uuid.replace(":", ""), "submission.xml") if not default_storage.exists(instance_path): - if self._get_response(self.download_submission_url, - params={'formId': form_str}): + if self._get_response( + self.download_submission_url, params={"formId": form_str} + ): instance_res = self._current_response content = instance_res.content.strip() default_storage.save(instance_path, ContentFile(content)) @@ -232,12 +232,12 @@ def download_instances(self, form_id, cursor=0, num_entries=100): except ExpatError: continue - media_path = os.path.join(path, uuid.replace(':', '')) + media_path = os.path.join(path, uuid.replace(":", "")) self.download_media_files(instance_doc, media_path) self.logger.debug("Fetched %s %s submission" % (form_id, uuid)) - if xml_doc.getElementsByTagName('resumptionCursor'): - rs_node = xml_doc.getElementsByTagName('resumptionCursor')[0] + if xml_doc.getElementsByTagName("resumptionCursor"): + rs_node = xml_doc.getElementsByTagName("resumptionCursor")[0] cursor = rs_node.childNodes[0].nodeValue if self.resumption_cursor != cursor: @@ -258,19 +258,20 @@ def _upload_instance(self, xml_file, instance_dir_path, files): de_node = xml_doc.documentElement for node in de_node.firstChild.childNodes: xml.write(node.toxml()) - new_xml_file = ContentFile(xml.getvalue().encode('utf-8')) - new_xml_file.content_type = 'text/xml' + new_xml_file = ContentFile(xml.getvalue().encode("utf-8")) + new_xml_file.content_type = "text/xml" xml.close() attachments = [] - for attach in de_node.getElementsByTagName('mediaFile'): - filename_node = attach.getElementsByTagName('filename') + for attach in de_node.getElementsByTagName("mediaFile"): + filename_node = attach.getElementsByTagName("filename") filename = filename_node[0].childNodes[0].nodeValue if filename in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, filename)) + os.path.join(instance_dir_path, filename) + ) mimetype, encoding = mimetypes.guess_type(file_obj.name) - media_obj = django_file(file_obj, 'media_files[]', mimetype) + media_obj = django_file(file_obj, "media_files[]", mimetype) attachments.append(media_obj) create_instance(self.user.username, new_xml_file, attachments) @@ -284,9 +285,10 @@ def _upload_instances(self, path): i_dirs, files = default_storage.listdir(instance_dir_path) xml_file = None - if 'submission.xml' in files: + if "submission.xml" in files: file_obj = default_storage.open( - os.path.join(instance_dir_path, 'submission.xml')) + os.path.join(instance_dir_path, "submission.xml") + ) xml_file = file_obj if xml_file: @@ -295,9 +297,12 @@ def _upload_instances(self, path): except ExpatError: continue except Exception as e: - logging.exception(_( - u'Ignoring exception, processing XML submission ' - 'raised exception: %s' % str(e))) + logging.exception( + _( + "Ignoring exception, processing XML submission " + "raised exception: %s" % str(e) + ) + ) else: instances_count += 1 @@ -308,7 +313,7 @@ def push(self): for form_dir in dirs: dir_path = os.path.join(self.forms_path, form_dir) form_dirs, form_files = default_storage.listdir(dir_path) - form_xml = '%s.xml' % form_dir + form_xml = "%s.xml" % form_dir if form_xml in form_files: form_xml_path = os.path.join(dir_path, form_xml) x = self._upload_xform(form_xml_path, form_xml) @@ -316,8 +321,7 @@ def push(self): self.logger.error("Failed to publish %s" % form_dir) else: self.logger.debug("Successfully published %s" % form_dir) - if 'instances' in form_dirs: + if "instances" in form_dirs: self.logger.debug("Uploading instances") - c = self._upload_instances(os.path.join(dir_path, 'instances')) - self.logger.debug("Published %d instances for %s" % - (c, form_dir)) + c = self._upload_instances(os.path.join(dir_path, "instances")) + self.logger.debug("Published %d instances for %s" % (c, form_dir)) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index f791d6fa6e..1dac617129 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -25,7 +25,7 @@ from django.core.files.storage import default_storage from django.utils import timezone from django.utils.translation import ugettext as _ -from future.utils import iteritems +from six import iteritems from multidb.pinning import use_master from onadata.apps.logger.models import Instance, XForm diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 89e1d19fa4..0d556d8efe 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,5 +1,5 @@ from functools import wraps -from future.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME from django.utils.decorators import available_attrs @@ -21,20 +21,22 @@ def is_owner(view_func): def _wrapped_view(request, *args, **kwargs): # assume username is first arg if request.user.is_authenticated: - if request.user.username == kwargs['username']: + if request.user.username == kwargs["username"]: return view_func(request, *args, **kwargs) protocol = "https" if request.is_secure() else "http" - return HttpResponseRedirect("%s://%s" % (protocol, - request.get_host())) + return HttpResponseRedirect("%s://%s" % (protocol, request.get_host())) path = request.build_absolute_uri() login_url = request.build_absolute_uri(settings.LOGIN_URL) # If the login url is the same scheme and net location then just # use the path as the "next" url. login_scheme, login_netloc = urlparse(login_url)[:2] current_scheme, current_netloc = urlparse(path)[:2] - if ((not login_scheme or login_scheme == current_scheme) and - (not login_netloc or login_netloc == current_netloc)): + if (not login_scheme or login_scheme == current_scheme) and ( + not login_netloc or login_netloc == current_netloc + ): path = request.get_full_path() from django.contrib.auth.views import redirect_to_login + return redirect_to_login(path, None, REDIRECT_FIELD_NAME) + return _wrapped_view diff --git a/onadata/libs/utils/email.py b/onadata/libs/utils/email.py index d67a9d4731..9b11a5ada7 100644 --- a/onadata/libs/utils/email.py +++ b/onadata/libs/utils/email.py @@ -1,84 +1,68 @@ from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string -from future.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from rest_framework.reverse import reverse def get_verification_url(redirect_url, request, verification_key): verification_url = getattr(settings, "VERIFICATION_URL", None) - url = verification_url or reverse( - 'userprofile-verify-email', request=request - ) - query_params_dict = {'verification_key': verification_key} - redirect_url and query_params_dict.update({ - 'redirect_url': redirect_url - }) + url = verification_url or reverse("userprofile-verify-email", request=request) + query_params_dict = {"verification_key": verification_key} + redirect_url and query_params_dict.update({"redirect_url": redirect_url}) query_params_string = urlencode(query_params_dict) - verification_url = '{}?{}'.format(url, query_params_string) + verification_url = "{}?{}".format(url, query_params_string) return verification_url def get_verification_email_data(email, username, verification_url, request): - email_data = {'email': email} + email_data = {"email": email} ctx_dict = { - 'username': username, - 'expiration_days': getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), - 'verification_url': verification_url + "username": username, + "expiration_days": getattr(settings, "ACCOUNT_ACTIVATION_DAYS", 1), + "verification_url": verification_url, } key_template_path_dict = { - 'subject': 'registration/verification_email_subject.txt', - 'message_txt': 'registration/verification_email.txt' + "subject": "registration/verification_email_subject.txt", + "message_txt": "registration/verification_email.txt", } for key, template_path in key_template_path_dict.items(): - email_data.update({ - key: render_to_string( - template_path, - ctx_dict, - request=request - ) - }) + email_data.update( + {key: render_to_string(template_path, ctx_dict, request=request)} + ) return email_data def get_account_lockout_email_data(username, ip, end=False): """Generates both the email upon start and end of account lockout""" - message_path = 'account_lockout/lockout_start.txt' - subject_path = 'account_lockout/lockout_email_subject.txt' + message_path = "account_lockout/lockout_start.txt" + subject_path = "account_lockout/lockout_email_subject.txt" if end: - message_path = 'account_lockout/lockout_end.txt' + message_path = "account_lockout/lockout_end.txt" ctx_dict = { - 'username': username, - 'remote_ip': ip, - 'lockout_time': getattr(settings, 'LOCKOUT_TIME', 1800) / 60, - 'support_email': getattr( - settings, 'SUPPORT_EMAIL', 'support@example.com') + "username": username, + "remote_ip": ip, + "lockout_time": getattr(settings, "LOCKOUT_TIME", 1800) / 60, + "support_email": getattr(settings, "SUPPORT_EMAIL", "support@example.com"), } email_data = { - 'subject': render_to_string(subject_path), - 'message_txt': render_to_string(message_path, ctx_dict) + "subject": render_to_string(subject_path), + "message_txt": render_to_string(message_path, ctx_dict), } return email_data def send_generic_email(email, message_txt, subject): - if any(a in [None, ''] for a in [email, message_txt, subject]): - raise ValueError( - "email, message_txt amd subject arguments are ALL required." - ) + if any(a in [None, ""] for a in [email, message_txt, subject]): + raise ValueError("email, message_txt amd subject arguments are ALL required.") from_email = settings.DEFAULT_FROM_EMAIL - email_message = EmailMultiAlternatives( - subject, - message_txt, - from_email, - [email] - ) + email_message = EmailMultiAlternatives(subject, message_txt, from_email, [email]) email_message.send() diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 9d0249737c..018b7f74ec 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -18,7 +18,7 @@ from django.core.files.temp import NamedTemporaryFile from onadata.libs.utils.common_tools import str_to_bool from django.utils.translation import ugettext as _ -from future.utils import iteritems +from six import iteritems from openpyxl.utils.datetime import to_excel from openpyxl.workbook import Workbook from pyxform.question import Question @@ -26,20 +26,44 @@ from savReaderWriter import SavWriter from onadata.apps.logger.models.osmdata import OsmData -from onadata.apps.logger.models.xform import (QUESTION_TYPES_TO_EXCLUDE, - _encode_for_mongo) +from onadata.apps.logger.models.xform import ( + QUESTION_TYPES_TO_EXCLUDE, + _encode_for_mongo, +) from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.utils.common_tags import ( - ATTACHMENTS, BAMBOO_DATASET_ID, DELETEDAT, DURATION, GEOLOCATION, - ID, INDEX, MULTIPLE_SELECT_TYPE, SELECT_ONE, NOTES, PARENT_INDEX, - PARENT_TABLE_NAME, REPEAT_INDEX_TAGS, SAV_255_BYTES_TYPE, - SAV_NUMERIC_TYPE, STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, UUID, - VERSION, XFORM_ID_STRING, REVIEW_STATUS, REVIEW_COMMENT, SELECT_BIND_TYPE, - REVIEW_DATE) + ATTACHMENTS, + BAMBOO_DATASET_ID, + DELETEDAT, + DURATION, + GEOLOCATION, + ID, + INDEX, + MULTIPLE_SELECT_TYPE, + SELECT_ONE, + NOTES, + PARENT_INDEX, + PARENT_TABLE_NAME, + REPEAT_INDEX_TAGS, + SAV_255_BYTES_TYPE, + SAV_NUMERIC_TYPE, + STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + UUID, + VERSION, + XFORM_ID_STRING, + REVIEW_STATUS, + REVIEW_COMMENT, + SELECT_BIND_TYPE, + REVIEW_DATE, +) from onadata.libs.utils.mongo import _decode_from_mongo, _is_invalid_for_mongo + # the bind type of select multiples that we use to compare -GEOPOINT_BIND_TYPE = 'geopoint' -OSM_BIND_TYPE = 'osm' +GEOPOINT_BIND_TYPE = "geopoint" +OSM_BIND_TYPE = "osm" DEFAULT_UPDATE_BATCH = 100 YES = 1 @@ -56,14 +80,15 @@ def current_site_url(path): :return: complete url """ from django.contrib.sites.models import Site + current_site = Site.objects.get_current() - protocol = getattr(settings, 'ONA_SITE_PROTOCOL', 'http') - port = getattr(settings, 'ONA_SITE_PORT', '') - url = '%s://%s' % (protocol, current_site.domain) + protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") + port = getattr(settings, "ONA_SITE_PORT", "") + url = "%s://%s" % (protocol, current_site.domain) if port: - url += ':%s' % port + url += ":%s" % port if path: - url += '%s' % path + url += "%s" % path return url @@ -74,8 +99,11 @@ def get_choice_label(label, data_dictionary, language=None): """ if isinstance(label, dict): languages = [i for i in label.keys()] - _language = language if language in languages else \ - data_dictionary.get_language(languages) + _language = ( + language + if language in languages + else data_dictionary.get_language(languages) + ) return label[_language] @@ -87,12 +115,12 @@ def get_choice_label_value(key, value, data_dictionary, language=None): Return the label of a choice matching the value if the key xpath is a SELECT_ONE otherwise it returns the value unchanged. """ + def _get_choice_label_value(lookup): _label = None for choice in data_dictionary.get_survey_element(key).children: if choice.name == lookup: - _label = get_choice_label(choice.label, data_dictionary, - language) + _label = get_choice_label(choice.label, data_dictionary, language) break return _label @@ -103,27 +131,34 @@ def _get_choice_label_value(lookup): if key in data_dictionary.get_select_multiple_xpaths(): answers = [] - for item in value.split(' '): + for item in value.split(" "): answer = _get_choice_label_value(item) answers.append(answer or item) if [_i for _i in answers if _i is not None]: - label = ' '.join(answers) + label = " ".join(answers) return label or value def get_value_or_attachment_uri( # pylint: disable=too-many-arguments - key, value, row, data_dictionary, media_xpaths, - attachment_list=None, show_choice_labels=False, language=None): + key, + value, + row, + data_dictionary, + media_xpaths, + attachment_list=None, + show_choice_labels=False, + language=None, +): """ - Gets either the attachment value or the attachment url - :param key: used to retrieve survey element - :param value: filename - :param row: current records row - :param data_dictionary: form structure - :param include_images: boolean value to either inlcude images or not - :param attachment_list: to be used incase row doesn't have ATTACHMENTS key - :return: value + Gets either the attachment value or the attachment url + :param key: used to retrieve survey element + :param value: filename + :param row: current records row + :param data_dictionary: form structure + :param include_images: boolean value to either inlcude images or not + :param attachment_list: to be used incase row doesn't have ATTACHMENTS key + :return: value """ if show_choice_labels: value = get_choice_label_value(key, value, data_dictionary, language) @@ -135,10 +170,10 @@ def get_value_or_attachment_uri( # pylint: disable=too-many-arguments attachments = [ a for a in row.get(ATTACHMENTS, attachment_list or []) - if a.get('name') == value + if a.get("name") == value ] if attachments: - value = current_site_url(attachments[0].get('download_url', '')) + value = current_site_url(attachments[0].get("download_url", "")) return value @@ -156,24 +191,24 @@ def encode_if_str(row, key, encode_dates=False, sav_writer=None): if sav_writer: if isinstance(val, datetime): if len(val.isoformat()): - strptime_fmt = '%Y-%m-%dT%H:%M:%S' + strptime_fmt = "%Y-%m-%dT%H:%M:%S" else: - strptime_fmt = '%Y-%m-%dT%H:%M:%S.%f%z' + strptime_fmt = "%Y-%m-%dT%H:%M:%S.%f%z" else: - strptime_fmt = '%Y-%m-%d' - return sav_writer.spssDateTime(val.isoformat().encode('utf-8'), - strptime_fmt) + strptime_fmt = "%Y-%m-%d" + return sav_writer.spssDateTime( + val.isoformat().encode("utf-8"), strptime_fmt + ) elif encode_dates: return val.isoformat() if sav_writer: - val = '' if val is None else val + val = "" if val is None else val return text(val) if IS_PY_3K and not isinstance(val, bool) else val return val -def dict_to_joined_export(data, index, indices, name, survey, row, - media_xpaths=[]): +def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths=[]): """ Converts a dict into one or more tabular datasets :param data: current record which can be changed or updated @@ -195,10 +230,13 @@ def dict_to_joined_export(data, index, indices, name, survey, row, indices[key] += 1 child_index = indices[key] new_output = dict_to_joined_export( - child, child_index, indices, key, survey, row, - media_xpaths) - d = {INDEX: child_index, PARENT_INDEX: index, - PARENT_TABLE_NAME: name} + child, child_index, indices, key, survey, row, media_xpaths + ) + d = { + INDEX: child_index, + PARENT_INDEX: index, + PARENT_TABLE_NAME: name, + } # iterate over keys within new_output and append to # main output for (out_key, out_val) in iteritems(new_output): @@ -213,16 +251,20 @@ def dict_to_joined_export(data, index, indices, name, survey, row, if name not in output: output[name] = {} if key in [TAGS]: - output[name][key] = ','.join(val) + output[name][key] = ",".join(val) elif key in [NOTES]: - note_list = [v if isinstance(v, text) - else v['note'] for v in val] - output[name][key] = '\r\n'.join(note_list) + note_list = [v if isinstance(v, text) else v["note"] for v in val] + output[name][key] = "\r\n".join(note_list) else: data_dictionary = get_data_dictionary_from_survey(survey) output[name][key] = get_value_or_attachment_uri( - key, val, data, data_dictionary, media_xpaths, - row and row.get(ATTACHMENTS)) + key, + val, + data, + data_dictionary, + media_xpaths, + row and row.get(ATTACHMENTS), + ) return output @@ -239,16 +281,22 @@ def is_all_numeric(items): for i in items: float(i) # if there is a zero padded number, it is not all numeric - if isinstance(i, text) and len(i) > 1 and \ - i[0] == '0' and i[1] != '.': + if isinstance(i, text) and len(i) > 1 and i[0] == "0" and i[1] != ".": return False return True except ValueError: return False # check for zero padded numbers to be treated as non numeric - return not (any([i.startswith('0') and len(i) > 1 and i.find('.') == -1 - for i in items if isinstance(i, text)])) + return not ( + any( + [ + i.startswith("0") and len(i) > 1 and i.find(".") == -1 + for i in items + if isinstance(i, text) + ] + ) + ) def track_task_progress(additions, total=None): @@ -261,19 +309,23 @@ def track_task_progress(additions, total=None): :return: """ try: - if additions % getattr(settings, 'EXPORT_TASK_PROGRESS_UPDATE_BATCH', - DEFAULT_UPDATE_BATCH) == 0: - meta = {'progress': additions} + if ( + additions + % getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH + ) + == 0 + ): + meta = {"progress": additions} if total: - meta.update({'total': total}) - current_task.update_state(state='PROGRESS', meta=meta) + meta.update({"total": total}) + current_task.update_state(state="PROGRESS", meta=meta) except Exception as e: - logging.exception( - _('Track task progress threw exception: %s' % text(e))) + logging.exception(_("Track task progress threw exception: %s" % text(e))) def string_to_date_with_xls_validation(date_str): - """ Try to convert a string to a date object. + """Try to convert a string to a date object. :param date_str: string to convert :returns: object if converted, otherwise date string @@ -282,7 +334,7 @@ def string_to_date_with_xls_validation(date_str): return date_str try: - date_obj = datetime.strptime(date_str, '%Y-%m-%d').date() + date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() to_excel(date_obj) except ValueError: return date_str @@ -291,7 +343,7 @@ def string_to_date_with_xls_validation(date_str): def decode_mongo_encoded_section_names(data): - """ Recursively decode mongo keys. + """Recursively decode mongo keys. :param data: A dictionary to decode. """ @@ -301,32 +353,49 @@ def decode_mongo_encoded_section_names(data): if isinstance(v, dict): new_v = decode_mongo_encoded_section_names(v) elif isinstance(v, list): - new_v = [decode_mongo_encoded_section_names(x) - if isinstance(x, dict) else x for x in v] + new_v = [ + decode_mongo_encoded_section_names(x) if isinstance(x, dict) else x + for x in v + ] results[_decode_from_mongo(k)] = new_v return results class ExportBuilder(object): - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - BAMBOO_DATASET_ID, DELETEDAT] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + BAMBOO_DATASET_ID, + DELETEDAT, + ] # fields we export but are not within the form's structure EXTRA_FIELDS = [ - ID, UUID, SUBMISSION_TIME, INDEX, PARENT_TABLE_NAME, PARENT_INDEX, - TAGS, NOTES, VERSION, DURATION, - SUBMITTED_BY] + ID, + UUID, + SUBMISSION_TIME, + INDEX, + PARENT_TABLE_NAME, + PARENT_INDEX, + TAGS, + NOTES, + VERSION, + DURATION, + SUBMITTED_BY, + ] SPLIT_SELECT_MULTIPLES = True BINARY_SELECT_MULTIPLES = False VALUE_SELECT_MULTIPLES = False # column group delimiters get_value_or_attachment_uri - GROUP_DELIMITER_SLASH = '/' - GROUP_DELIMITER_DOT = '.' + GROUP_DELIMITER_SLASH = "/" + GROUP_DELIMITER_DOT = "." GROUP_DELIMITER = GROUP_DELIMITER_SLASH GROUP_DELIMITERS = [GROUP_DELIMITER_SLASH, GROUP_DELIMITER_DOT] # index tags - REPEAT_INDEX_TAGS = ('[', ']') + REPEAT_INDEX_TAGS = ("[", "]") INCLUDE_LABELS = False INCLUDE_LABELS_ONLY = False @@ -336,12 +405,12 @@ class ExportBuilder(object): SHOW_CHOICE_LABELS = False INCLUDE_REVIEWS = False - TYPES_TO_CONVERT = ['int', 'decimal', 'date'] # , 'dateTime'] + TYPES_TO_CONVERT = ["int", "decimal", "date"] # , 'dateTime'] CONVERT_FUNCS = { - 'int': int, - 'decimal': float, - 'date': string_to_date_with_xls_validation, - 'dateTime': lambda x: datetime.strptime(x[:19], '%Y-%m-%dT%H:%M:%S') + "int": int, + "decimal": float, + "date": string_to_date_with_xls_validation, + "dateTime": lambda x: datetime.strptime(x[:19], "%Y-%m-%dT%H:%M:%S"), } TRUNCATE_GROUP_TITLE = False @@ -351,13 +420,17 @@ class ExportBuilder(object): language = None def __init__(self): - self.extra_columns = ( - self.EXTRA_FIELDS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.extra_columns = self.EXTRA_FIELDS + getattr(settings, "EXTRA_COLUMNS", []) self.osm_columns = [] @classmethod - def format_field_title(cls, abbreviated_xpath, field_delimiter, - data_dictionary, remove_group_name=False): + def format_field_title( + cls, + abbreviated_xpath, + field_delimiter, + data_dictionary, + remove_group_name=False, + ): title = abbreviated_xpath # Check if to truncate the group name prefix if remove_group_name: @@ -365,13 +438,13 @@ def format_field_title(cls, abbreviated_xpath, field_delimiter, # incase abbreviated_xpath is a choices xpath if elem is None: pass - elif elem.type == '': - title = '/'.join([elem.parent.name, elem.name]) + elif elem.type == "": + title = "/".join([elem.parent.name, elem.name]) else: title = elem.name - if field_delimiter != '/': - title = field_delimiter.join(title.split('/')) + if field_delimiter != "/": + title = field_delimiter.join(title.split("/")) return title @@ -382,170 +455,232 @@ def get_choice_label_from_dict(self, label): return label - def _get_select_mulitples_choices(self, child, dd, field_delimiter, - remove_group_name): + def _get_select_mulitples_choices( + self, child, dd, field_delimiter, remove_group_name + ): def get_choice_dict(xpath, label): title = ExportBuilder.format_field_title( xpath, field_delimiter, dd, remove_group_name ) return { - 'label': field_delimiter.join([child.name, label or title]), - '_label': label or title, - '_label_xpath': field_delimiter.join([child.name, - label or title]), - 'title': title, - 'xpath': xpath, - 'type': 'string' + "label": field_delimiter.join([child.name, label or title]), + "_label": label or title, + "_label_xpath": field_delimiter.join([child.name, label or title]), + "title": title, + "xpath": xpath, + "type": "string", } choices = [] is_choice_randomized = str_to_bool( - child.parameters and child.parameters.get('randomize')) - if ((not child.children and child.choice_filter) - or is_choice_randomized) and child.itemset: - itemset = dd.survey.to_json_dict()['choices'].get(child.itemset) - choices = [get_choice_dict( - '/'.join([child.get_abbreviated_xpath(), i['name']]), - self.get_choice_label_from_dict(i['label']) - ) for i in itemset] if itemset else choices + child.parameters and child.parameters.get("randomize") + ) + if ( + (not child.children and child.choice_filter) or is_choice_randomized + ) and child.itemset: + itemset = dd.survey.to_json_dict()["choices"].get(child.itemset) + choices = ( + [ + get_choice_dict( + "/".join([child.get_abbreviated_xpath(), i["name"]]), + self.get_choice_label_from_dict(i["label"]), + ) + for i in itemset + ] + if itemset + else choices + ) else: - choices = [get_choice_dict( - c.get_abbreviated_xpath(), - get_choice_label(c.label, dd, language=self.language)) - for c in child.children] + choices = [ + get_choice_dict( + c.get_abbreviated_xpath(), + get_choice_label(c.label, dd, language=self.language), + ) + for c in child.children + ] return choices def set_survey(self, survey, xform=None, include_reviews=False): if self.INCLUDE_REVIEWS or include_reviews: self.EXTRA_FIELDS = self.EXTRA_FIELDS + [ - REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE] + REVIEW_STATUS, + REVIEW_COMMENT, + REVIEW_DATE, + ] self.__init__() dd = get_data_dictionary_from_survey(survey) def build_sections( - current_section, survey_element, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, select_ones, - field_delimiter='/', remove_group_name=False, language=None): + current_section, + survey_element, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter="/", + remove_group_name=False, + language=None, + ): for child in survey_element.children: - current_section_name = current_section['name'] + current_section_name = current_section["name"] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { - 'name': child.get_abbreviated_xpath(), - 'elements': []} + "name": child.get_abbreviated_xpath(), + "elements": [], + } self.sections.append(section) build_sections( - section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) + section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) else: # its a group, recurs using the same section build_sections( - current_section, child, sections, select_multiples, - gps_fields, osm_fields, encoded_fields, - select_ones, field_delimiter, remove_group_name, - language=language) - elif isinstance(child, Question) and \ - (child.bind.get('type') - not in QUESTION_TYPES_TO_EXCLUDE and - child.type not in QUESTION_TYPES_TO_EXCLUDE): + current_section, + child, + sections, + select_multiples, + gps_fields, + osm_fields, + encoded_fields, + select_ones, + field_delimiter, + remove_group_name, + language=language, + ) + elif isinstance(child, Question) and ( + child.bind.get("type") not in QUESTION_TYPES_TO_EXCLUDE + and child.type not in QUESTION_TYPES_TO_EXCLUDE + ): # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( child.get_abbreviated_xpath(), - field_delimiter, dd, remove_group_name + field_delimiter, + dd, + remove_group_name, + ) + _label = ( + dd.get_label(child_xpath, elem=child, language=language) + or _title + ) + current_section["elements"].append( + { + "label": _label, + "title": _title, + "xpath": child_xpath, + "type": child.bind.get("type"), + } ) - _label = \ - dd.get_label( - child_xpath, - elem=child, - language=language) or _title - current_section['elements'].append({ - 'label': _label, - 'title': _title, - 'xpath': child_xpath, - 'type': child.bind.get('type') - }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( - {child_xpath: _encode_for_mongo(child_xpath)}) + {child_xpath: _encode_for_mongo(child_xpath)} + ) # if its a select multiple, make columns out of its choices - if child.bind.get('type') == SELECT_BIND_TYPE \ - and child.type == MULTIPLE_SELECT_TYPE: + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == MULTIPLE_SELECT_TYPE + ): choices = [] if self.SPLIT_SELECT_MULTIPLES: choices = self._get_select_mulitples_choices( child, dd, field_delimiter, remove_group_name ) for choice in choices: - if choice not in current_section['elements']: - current_section['elements'].append(choice) + if choice not in current_section["elements"]: + current_section["elements"].append(choice) # choices_xpaths = [c['xpath'] for c in choices] _append_xpaths_to_section( - current_section_name, select_multiples, - child.get_abbreviated_xpath(), choices) + current_section_name, + select_multiples, + child.get_abbreviated_xpath(), + choices, + ) # split gps fields within this section - if child.bind.get('type') == GEOPOINT_BIND_TYPE: + if child.bind.get("type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( - child.get_abbreviated_xpath()) + child.get_abbreviated_xpath() + ) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, field_delimiter, dd, remove_group_name + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "decimal", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'decimal' - }) _append_xpaths_to_section( - current_section_name, gps_fields, - child.get_abbreviated_xpath(), xpaths) + current_section_name, + gps_fields, + child.get_abbreviated_xpath(), + xpaths, + ) # get other osm fields - if child.get(u"type") == OSM_BIND_TYPE: + if child.get("type") == OSM_BIND_TYPE: xpaths = _get_osm_paths(child, xform) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, - remove_group_name + xpath, field_delimiter, dd, remove_group_name + ) + current_section["elements"].append( + { + "label": _title, + "title": _title, + "xpath": xpath, + "type": "osm", + } ) - current_section['elements'].append({ - 'label': _title, - 'title': _title, - 'xpath': xpath, - 'type': 'osm' - }) _append_xpaths_to_section( - current_section_name, osm_fields, - child.get_abbreviated_xpath(), xpaths) - if child.bind.get(u"type") == SELECT_BIND_TYPE \ - and child.type == SELECT_ONE: + current_section_name, + osm_fields, + child.get_abbreviated_xpath(), + xpaths, + ) + if ( + child.bind.get("type") == SELECT_BIND_TYPE + and child.type == SELECT_ONE + ): _append_xpaths_to_section( - current_section_name, select_ones, - child.get_abbreviated_xpath(), []) + current_section_name, + select_ones, + child.get_abbreviated_xpath(), + [], + ) - def _append_xpaths_to_section(current_section_name, field_list, xpath, - xpaths): + def _append_xpaths_to_section(current_section_name, field_list, xpath, xpaths): if current_section_name not in field_list: field_list[current_section_name] = {} - field_list[ - current_section_name][xpath] = xpaths + field_list[current_section_name][xpath] = xpaths def _get_osm_paths(osm_field, xform): """ @@ -555,8 +690,8 @@ def _get_osm_paths(osm_field, xform): osm_columns = [] if osm_field and xform: osm_columns = OsmData.get_tag_keys( - xform, osm_field.get_abbreviated_xpath(), - include_prefix=True) + xform, osm_field.get_abbreviated_xpath(), include_prefix=True + ) return osm_columns self.dd = dd @@ -566,27 +701,40 @@ def _get_osm_paths(osm_field, xform): self.gps_fields = {} self.osm_fields = {} self.encoded_fields = {} - main_section = {'name': survey.name, 'elements': []} + main_section = {"name": survey.name, "elements": []} self.sections = [main_section] build_sections( - main_section, self.survey, self.sections, - self.select_multiples, self.gps_fields, self.osm_fields, - self.encoded_fields, self.select_ones, self.GROUP_DELIMITER, - self.TRUNCATE_GROUP_TITLE, language=self.language) + main_section, + self.survey, + self.sections, + self.select_multiples, + self.gps_fields, + self.osm_fields, + self.encoded_fields, + self.select_ones, + self.GROUP_DELIMITER, + self.TRUNCATE_GROUP_TITLE, + language=self.language, + ) def section_by_name(self, name): - matches = [s for s in self.sections if s['name'] == name] - assert(len(matches) == 1) + matches = [s for s in self.sections if s["name"] == name] + assert len(matches) == 1 return matches[0] # pylint: disable=too-many-arguments @classmethod - def split_select_multiples(cls, row, select_multiples, - select_values=False, - binary_select_multiples=False, - show_choice_labels=False, data_dictionary=None, - language=None): + def split_select_multiples( + cls, + row, + select_multiples, + select_values=False, + binary_select_multiples=False, + show_choice_labels=False, + data_dictionary=None, + language=None, + ): """ Split select multiple choices in a submission to individual columns. @@ -613,37 +761,69 @@ def split_select_multiples(cls, row, select_multiples, selections = [] if data: selections = [ - '{0}/{1}'.format( - xpath, selection) for selection in data.split()] + "{0}/{1}".format(xpath, selection) for selection in data.split() + ] if show_choice_labels and data_dictionary: row[xpath] = get_choice_label_value( - xpath, data, data_dictionary, language) + xpath, data, data_dictionary, language + ) if select_values: if show_choice_labels: - row.update(dict( - [(choice['label'], choice['_label'] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"], + choice["_label"] + if selections and choice["xpath"] in selections + else None, + ) + for choice in choices + ] + ) + ) else: - row.update(dict( - [(choice['xpath'], - data.split()[selections.index(choice['xpath'])] - if selections and choice['xpath'] in selections - else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["xpath"], + data.split()[selections.index(choice["xpath"])] + if selections and choice["xpath"] in selections + else None, + ) + for choice in choices + ] + ) + ) elif binary_select_multiples: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - YES if choice['xpath'] in selections else NO) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"] + if show_choice_labels + else choice["xpath"], + YES if choice["xpath"] in selections else NO, + ) + for choice in choices + ] + ) + ) else: - row.update(dict( - [(choice['label'] - if show_choice_labels else choice['xpath'], - choice['xpath'] in selections if selections else None) - for choice in choices])) + row.update( + dict( + [ + ( + choice["label"] + if show_choice_labels + else choice["xpath"], + choice["xpath"] in selections if selections else None, + ) + for choice in choices + ] + ) + ) return row @classmethod @@ -683,61 +863,70 @@ def pre_process_row(self, row, section): """ Split select multiples, gps and decode . and $ """ - section_name = section['name'] + section_name = section["name"] # first decode fields so that subsequent lookups # have decoded field names if section_name in self.encoded_fields: row = ExportBuilder.decode_mongo_encoded_fields( - row, self.encoded_fields[section_name]) + row, self.encoded_fields[section_name] + ) if section_name in self.select_multiples: select_multiples = self.select_multiples[section_name] if self.SPLIT_SELECT_MULTIPLES: row = ExportBuilder.split_select_multiples( - row, select_multiples, self.VALUE_SELECT_MULTIPLES, + row, + select_multiples, + self.VALUE_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, show_choice_labels=self.SHOW_CHOICE_LABELS, - data_dictionary=self.dd, language=self.language) + data_dictionary=self.dd, + language=self.language, + ) if not self.SPLIT_SELECT_MULTIPLES and self.SHOW_CHOICE_LABELS: for xpath in select_multiples: # get the data matching this xpath data = row.get(xpath) and text(row.get(xpath)) if data: row[xpath] = get_choice_label_value( - xpath, data, self.dd, self.language) + xpath, data, self.dd, self.language + ) if section_name in self.gps_fields: - row = ExportBuilder.split_gps_components( - row, self.gps_fields[section_name]) + row = ExportBuilder.split_gps_components(row, self.gps_fields[section_name]) if section_name in self.select_ones and self.SHOW_CHOICE_LABELS: for key in self.select_ones[section_name]: if key in row: - row[key] = get_choice_label_value(key, row[key], self.dd, - self.language) + row[key] = get_choice_label_value( + key, row[key], self.dd, self.language + ) # convert to native types - for elm in section['elements']: + for elm in section["elements"]: # only convert if its in our list and its not empty, just to # optimize - value = row.get(elm['xpath']) - if elm['type'] in ExportBuilder.TYPES_TO_CONVERT\ - and value is not None and value != '': - row[elm['xpath']] = ExportBuilder.convert_type( - value, elm['type']) + value = row.get(elm["xpath"]) + if ( + elm["type"] in ExportBuilder.TYPES_TO_CONVERT + and value is not None + and value != "" + ): + row[elm["xpath"]] = ExportBuilder.convert_type(value, elm["type"]) if SUBMISSION_TIME in row: row[SUBMISSION_TIME] = ExportBuilder.convert_type( - row[SUBMISSION_TIME], 'dateTime') + row[SUBMISSION_TIME], "dateTime" + ) # Map dynamic values for key, value in row.items(): if isinstance(value, str): - dynamic_val_regex = '\$\{\w+\}' # noqa + dynamic_val_regex = "\$\{\w+\}" # noqa # Find substrings that match ${`any_text`} result = re.findall(dynamic_val_regex, value) if result: for val in result: - val_key = val.replace('${', '').replace('}', '') + val_key = val.replace("${", "").replace("}", "") # Try retrieving value of ${`any_text`} from the # row data and replace the value if row.get(val_key): @@ -748,45 +937,41 @@ def pre_process_row(self, row, section): def to_zipped_csv(self, path, data, *args, **kwargs): def write_row(row, csv_writer, fields): - csv_writer.writerow( - [encode_if_str(row, field) for field in fields]) + csv_writer.writerow([encode_if_str(row, field) for field in fields]) csv_defs = {} - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") for section in self.sections: - csv_file = NamedTemporaryFile(suffix='.csv', mode='w') + csv_file = NamedTemporaryFile(suffix=".csv", mode="w") csv_writer = csv.writer(csv_file) - csv_defs[section['name']] = { - 'csv_file': csv_file, 'csv_writer': csv_writer} + csv_defs[section["name"]] = {"csv_file": csv_file, "csv_writer": csv_writer} # write headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "title") + csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - fields = self.get_fields(dataview, section, 'label') - csv_defs[section['name']]['csv_writer'].writerow( - [f for f in fields]) + fields = self.get_fields(dataview, section, "label") + csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") # write hxl row if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - fields = self.get_fields(dataview, section, 'title') - hxl_row = [columns_with_hxl.get(col, '') - for col in fields] + fields = self.get_fields(dataview, section, "title") + hxl_row = [columns_with_hxl.get(col, "") for col in fields] if hxl_row: - writer = csv_defs[section['name']]['csv_writer'] + writer = csv_defs[section["name"]]["csv_writer"] writer.writerow(hxl_row) index = 1 @@ -794,10 +979,9 @@ def write_row(row, csv_writer, fields): survey_name = self.survey.name for i, d in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -807,86 +991,82 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] csv_def = csv_defs[section_name] - fields = self.get_fields(dataview, section, 'xpath') - csv_writer = csv_def['csv_writer'] + fields = self.get_fields(dataview, section, "xpath") + csv_writer = csv_def["csv_writer"] # section name might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - csv_writer, fields) + write_row(self.pre_process_row(row, section), csv_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - csv_writer, fields) + self.pre_process_row(child_row, section), csv_writer, fields + ) index += 1 track_task_progress(i, total_records) # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, csv_def) in iteritems(csv_defs): - csv_file = csv_def['csv_file'] + csv_file = csv_def["csv_file"] csv_file.seek(0) zip_file.write( - csv_file.name, '_'.join(section_name.split('/')) + '.csv') + csv_file.name, "_".join(section_name.split("/")) + ".csv" + ) # close files when we are done for (section_name, csv_def) in iteritems(csv_defs): - csv_def['csv_file'].close() + csv_def["csv_file"].close() @classmethod def get_valid_sheet_name(cls, desired_name, existing_names): # a sheet name has to be <= 31 characters and not a duplicate of an # existing sheet # truncate sheet_name to XLSDataFrameBuilder.SHEET_NAME_MAX_CHARS - new_sheet_name = \ - desired_name[:cls.XLS_SHEET_NAME_MAX_CHARS] + new_sheet_name = desired_name[: cls.XLS_SHEET_NAME_MAX_CHARS] # make sure its unique within the list i = 1 generated_name = new_sheet_name while generated_name in existing_names: digit_length = len(text(i)) - allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - \ - digit_length + allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - digit_length # make name the required len if len(generated_name) > allowed_name_len: generated_name = generated_name[:allowed_name_len] - generated_name = '{0}{1}'.format(generated_name, i) + generated_name = "{0}{1}".format(generated_name, i) i += 1 return generated_name def to_xls_export(self, path, data, *args, **kwargs): def write_row(data, work_sheet, fields, work_sheet_titles): # update parent_table with the generated sheet's title - data[PARENT_TABLE_NAME] = work_sheet_titles.get( - data.get(PARENT_TABLE_NAME)) + data[PARENT_TABLE_NAME] = work_sheet_titles.get(data.get(PARENT_TABLE_NAME)) work_sheet.append([data.get(f) for f in fields]) - dataview = kwargs.get('dataview') - total_records = kwargs.get('total_records') + dataview = kwargs.get("dataview") + total_records = kwargs.get("total_records") wb = Workbook(write_only=True) work_sheets = {} # map of section_names to generated_names work_sheet_titles = {} for section in self.sections: - section_name = section['name'] + section_name = section["name"] work_sheet_title = ExportBuilder.get_valid_sheet_name( - '_'.join(section_name.split('/')), work_sheet_titles.values()) + "_".join(section_name.split("/")), work_sheet_titles.values() + ) work_sheet_titles[section_name] = work_sheet_title - work_sheets[section_name] = wb.create_sheet( - title=work_sheet_title) + work_sheets[section_name] = wb.create_sheet(title=work_sheet_title) # write the headers if not self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet ws = work_sheets[section_name] @@ -895,38 +1075,37 @@ def write_row(data, work_sheet, fields, work_sheet_titles): # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: - section_name = section['name'] - labels = self.get_fields(dataview, section, 'label') + section_name = section["name"] + labels = self.get_fields(dataview, section, "label") # get the worksheet ws = work_sheets[section_name] ws.append(labels) - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) # write hxl header - columns_with_hxl = kwargs.get('columns_with_hxl') + columns_with_hxl = kwargs.get("columns_with_hxl") if self.INCLUDE_HXL and columns_with_hxl: for section in self.sections: - section_name = section['name'] - headers = self.get_fields(dataview, section, 'title') + section_name = section["name"] + headers = self.get_fields(dataview, section, "title") # get the worksheet ws = work_sheets[section_name] - hxl_row = [columns_with_hxl.get(col, '') - for col in headers] + hxl_row = [columns_with_hxl.get(col, "") for col in headers] hxl_row and ws.append(hxl_row) index = 1 indices = {} survey_name = self.survey.name for i, d in enumerate(data, start=1): - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -936,8 +1115,8 @@ def write_row(data, work_sheet, fields, work_sheet_titles): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to xls - section_name = section['name'] - fields = self.get_fields(dataview, section, 'xpath') + section_name = section["name"] + fields = self.get_fields(dataview, section, "xpath") ws = work_sheets[section_name] # section might not exist within the output, e.g. data was @@ -946,60 +1125,79 @@ def write_row(data, work_sheet, fields, work_sheet_titles): if isinstance(row, dict): write_row( self.pre_process_row(row, section), - ws, fields, work_sheet_titles) + ws, + fields, + work_sheet_titles, + ) elif isinstance(row, list): for child_row in row: write_row( self.pre_process_row(child_row, section), - ws, fields, work_sheet_titles) + ws, + fields, + work_sheet_titles, + ) index += 1 track_task_progress(i, total_records) wb.save(filename=path) - def to_flat_csv_export(self, path, data, username, id_string, - filter_query, **kwargs): + def to_flat_csv_export( + self, path, data, username, id_string, filter_query, **kwargs + ): """ Generates a flattened CSV file for submitted data. """ # TODO resolve circular import from onadata.libs.utils.csv_builder import CSVDataFrameBuilder - start = kwargs.get('start') - end = kwargs.get('end') - dataview = kwargs.get('dataview') - xform = kwargs.get('xform') - options = kwargs.get('options') - total_records = kwargs.get('total_records') - win_excel_utf8 = options.get('win_excel_utf8') if options else False + + start = kwargs.get("start") + end = kwargs.get("end") + dataview = kwargs.get("dataview") + xform = kwargs.get("xform") + options = kwargs.get("options") + total_records = kwargs.get("total_records") + win_excel_utf8 = options.get("win_excel_utf8") if options else False index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS) - show_choice_labels = options.get('show_choice_labels', False) - language = options.get('language') + show_choice_labels = options.get("show_choice_labels", False) + language = options.get("language") csv_builder = CSVDataFrameBuilder( - username, id_string, filter_query, self.GROUP_DELIMITER, - self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, - start, end, self.TRUNCATE_GROUP_TITLE, xform, - self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, - self.INCLUDE_IMAGES, self.INCLUDE_HXL, - win_excel_utf8=win_excel_utf8, total_records=total_records, + username, + id_string, + filter_query, + self.GROUP_DELIMITER, + self.SPLIT_SELECT_MULTIPLES, + self.BINARY_SELECT_MULTIPLES, + start, + end, + self.TRUNCATE_GROUP_TITLE, + xform, + self.INCLUDE_LABELS, + self.INCLUDE_LABELS_ONLY, + self.INCLUDE_IMAGES, + self.INCLUDE_HXL, + win_excel_utf8=win_excel_utf8, + total_records=total_records, index_tags=index_tags, value_select_multiples=self.VALUE_SELECT_MULTIPLES, show_choice_labels=show_choice_labels, - include_reviews=self.INCLUDE_REVIEWS, language=language) + include_reviews=self.INCLUDE_REVIEWS, + language=language, + ) csv_builder.export_to(path, dataview=dataview) def get_default_language(self, languages): language = self.dd.default_language - if languages and \ - ((language and language not in languages) or not language): + if languages and ((language and language not in languages) or not language): languages.sort() language = languages[0] return language def _get_sav_value_labels(self, xpath_var_names=None): - """ GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form + """GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form `{varName: {value: valueLabel}}`: .. code-block: python @@ -1013,33 +1211,35 @@ def _get_sav_value_labels(self, xpath_var_names=None): sav_value_labels = {} for q in choice_questions: - if (xpath_var_names and - q.get_abbreviated_xpath() not in xpath_var_names): + if xpath_var_names and q.get_abbreviated_xpath() not in xpath_var_names: continue - var_name = xpath_var_names.get(q.get_abbreviated_xpath()) if \ - xpath_var_names else q['name'] - choices = q.to_json_dict().get('children') + var_name = ( + xpath_var_names.get(q.get_abbreviated_xpath()) + if xpath_var_names + else q["name"] + ) + choices = q.to_json_dict().get("children") if choices is None: - choices = self.survey.get('choices') - if choices is not None and q.get('itemset'): - choices = choices.get(q.get('itemset')) + choices = self.survey.get("choices") + if choices is not None and q.get("itemset"): + choices = choices.get(q.get("itemset")) _value_labels = {} if choices: - is_numeric = is_all_numeric([c['name'] for c in choices]) + is_numeric = is_all_numeric([c["name"] for c in choices]) for choice in choices: - name = choice['name'].strip() + name = choice["name"].strip() # should skip select multiple and zero padded numbers e.g # 009 or 09, they should be treated as strings - if q.type != 'select all that apply' and is_numeric: + if q.type != "select all that apply" and is_numeric: try: - name = float(name) \ - if (float(name) > int(name)) else int(name) + name = ( + float(name) if (float(name) > int(name)) else int(name) + ) except ValueError: pass - label = self.get_choice_label_from_dict( - choice.get("label", "")) + label = self.get_choice_label_from_dict(choice.get("label", "")) _value_labels[name] = label.strip() - sav_value_labels[var_name or q['name']] = _value_labels + sav_value_labels[var_name or q["name"]] = _value_labels return sav_value_labels @@ -1050,10 +1250,15 @@ def _get_var_name(self, title, var_names): @param var_names - list of existing var_names @return valid varName and list of var_names with new var name appended """ - var_name = title.replace('/', '.').replace('-', '_'). \ - replace(':', '_').replace('{', '').replace('}', '') + var_name = ( + title.replace("/", ".") + .replace("-", "_") + .replace(":", "_") + .replace("{", "") + .replace("}", "") + ) var_name = self._check_sav_column(var_name, var_names) - var_name = '@' + var_name if var_name.startswith('_') else var_name + var_name = "@" + var_name if var_name.startswith("_") else var_name var_names.append(var_name) return var_name, var_names @@ -1071,11 +1276,12 @@ def _get_sav_options(self, elements): 'ioUtf8': True } """ + def _is_numeric(xpath, element_type, data_dictionary): var_name = xpath_var_names.get(xpath) or xpath - if element_type in ['decimal', 'int', 'date']: + if element_type in ["decimal", "int", "date"]: return True - elif element_type == 'string': + elif element_type == "string": # check if it is a choice part of multiple choice # type is likely empty string, split multi select is binary element = data_dictionary.get_element(xpath) @@ -1087,11 +1293,11 @@ def _is_numeric(xpath, element_type, data_dictionary): if len(choices) == 0: return False return is_all_numeric(choices) - if element and element.type == '' and value_select_multiples: + if element and element.type == "" and value_select_multiples: return is_all_numeric([element.name]) - parent_xpath = '/'.join(xpath.split('/')[:-1]) + parent_xpath = "/".join(xpath.split("/")[:-1]) parent = data_dictionary.get_element(parent_xpath) - return (parent and parent.type == MULTIPLE_SELECT_TYPE) + return parent and parent.type == MULTIPLE_SELECT_TYPE else: return False @@ -1102,18 +1308,20 @@ def _is_numeric(xpath, element_type, data_dictionary): var_names = [] fields_and_labels = [] - elements += [{'title': f, "label": f, "xpath": f, 'type': f} - for f in self.extra_columns] + elements += [ + {"title": f, "label": f, "xpath": f, "type": f} for f in self.extra_columns + ] for element in elements: - title = element['title'] + title = element["title"] _var_name, _var_names = self._get_var_name(title, var_names) var_names = _var_names - fields_and_labels.append((element['title'], element['label'], - element['xpath'], _var_name)) + fields_and_labels.append( + (element["title"], element["label"], element["xpath"], _var_name) + ) - xpath_var_names = dict([(xpath, var_name) - for field, label, xpath, var_name - in fields_and_labels]) + xpath_var_names = dict( + [(xpath, var_name) for field, label, xpath, var_name in fields_and_labels] + ) all_value_labels = self._get_sav_value_labels(xpath_var_names) @@ -1138,36 +1346,49 @@ def _get_element_type(element_xpath): return "" var_types = dict( - [(_var_types[element['xpath']], - SAV_NUMERIC_TYPE if _is_numeric(element['xpath'], - element['type'], - self.dd) else - SAV_255_BYTES_TYPE) - for element in elements] + - [(_var_types[item], - SAV_NUMERIC_TYPE if item in [ - '_id', '_index', '_parent_index', SUBMISSION_TIME] - else SAV_255_BYTES_TYPE) - for item in self.extra_columns] + - [(x[1], - SAV_NUMERIC_TYPE if _is_numeric( - x[0], - _get_element_type(x[0]), - self.dd) else SAV_255_BYTES_TYPE) - for x in duplicate_names] + [ + ( + _var_types[element["xpath"]], + SAV_NUMERIC_TYPE + if _is_numeric(element["xpath"], element["type"], self.dd) + else SAV_255_BYTES_TYPE, + ) + for element in elements + ] + + [ + ( + _var_types[item], + SAV_NUMERIC_TYPE + if item in ["_id", "_index", "_parent_index", SUBMISSION_TIME] + else SAV_255_BYTES_TYPE, + ) + for item in self.extra_columns + ] + + [ + ( + x[1], + SAV_NUMERIC_TYPE + if _is_numeric(x[0], _get_element_type(x[0]), self.dd) + else SAV_255_BYTES_TYPE, + ) + for x in duplicate_names + ] ) - dates = [_var_types[element['xpath']] for element in elements - if element.get('type') == 'date'] - formats = {d: 'EDATE40' for d in dates} - formats['@' + SUBMISSION_TIME] = 'DATETIME40' + dates = [ + _var_types[element["xpath"]] + for element in elements + if element.get("type") == "date" + ] + formats = {d: "EDATE40" for d in dates} + formats["@" + SUBMISSION_TIME] = "DATETIME40" return { - 'formats': formats, - 'varLabels': var_labels, - 'varNames': var_names, - 'varTypes': var_types, - 'valueLabels': value_labels, - 'ioUtf8': True + "formats": formats, + "varLabels": var_labels, + "varNames": var_names, + "varTypes": var_types, + "valueLabels": value_labels, + "ioUtf8": True, } def _check_sav_column(self, column, columns): @@ -1185,43 +1406,43 @@ def _check_sav_column(self, column, columns): if column.lower() in (t.lower() for t in columns): if len(column) > 59: column = column[:-5] - column = column + '@' + text(uuid.uuid4()).split('-')[1] + column = column + "@" + text(uuid.uuid4()).split("-")[1] return column def to_zipped_sav(self, path, data, *args, **kwargs): - total_records = kwargs.get('total_records') + total_records = kwargs.get("total_records") def write_row(row, csv_writer, fields): # replace character for osm fields - fields = [field.replace(':', '_') for field in fields] + fields = [field.replace(":", "_") for field in fields] sav_writer.writerow( - [encode_if_str(row, field, sav_writer=sav_writer) - for field in fields]) + [encode_if_str(row, field, sav_writer=sav_writer) for field in fields] + ) sav_defs = {} # write headers for section in self.sections: - sav_options = self._get_sav_options(section['elements']) - sav_file = NamedTemporaryFile(suffix='.sav') - sav_writer = SavWriter(sav_file.name, ioLocale=str('en_US.UTF-8'), - **sav_options) - sav_defs[section['name']] = { - 'sav_file': sav_file, 'sav_writer': sav_writer} + sav_options = self._get_sav_options(section["elements"]) + sav_file = NamedTemporaryFile(suffix=".sav") + sav_writer = SavWriter( + sav_file.name, ioLocale=str("en_US.UTF-8"), **sav_options + ) + sav_defs[section["name"]] = {"sav_file": sav_file, "sav_writer": sav_writer} - media_xpaths = [] if not self.INCLUDE_IMAGES \ - else self.dd.get_media_survey_xpaths() + media_xpaths = ( + [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + ) index = 1 indices = {} survey_name = self.survey.name for i, d in enumerate(data, start=1): # decode mongo section names - joined_export = dict_to_joined_export(d, index, indices, - survey_name, - self.survey, d, - media_xpaths) + joined_export = dict_to_joined_export( + d, index, indices, survey_name, self.survey, d, media_xpaths + ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section @@ -1231,52 +1452,53 @@ def write_row(row, csv_writer, fields): output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv - section_name = section['name'] + section_name = section["name"] sav_def = sav_defs[section_name] - fields = [ - element['xpath'] for element in - section['elements']] - sav_writer = sav_def['sav_writer'] + fields = [element["xpath"] for element in section["elements"]] + sav_writer = sav_def["sav_writer"] row = output.get(section_name, None) if isinstance(row, dict): - write_row( - self.pre_process_row(row, section), - sav_writer, fields) + write_row(self.pre_process_row(row, section), sav_writer, fields) elif isinstance(row, list): for child_row in row: write_row( - self.pre_process_row(child_row, section), - sav_writer, fields) + self.pre_process_row(child_row, section), sav_writer, fields + ) index += 1 track_task_progress(i, total_records) for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_writer'].closeSavFile( - sav_def['sav_writer'].fh, mode='wb') + sav_def["sav_writer"].closeSavFile(sav_def["sav_writer"].fh, mode="wb") # write zipfile - with ZipFile(path, 'w', ZIP_DEFLATED, allowZip64=True) as zip_file: + with ZipFile(path, "w", ZIP_DEFLATED, allowZip64=True) as zip_file: for (section_name, sav_def) in iteritems(sav_defs): - sav_file = sav_def['sav_file'] + sav_file = sav_def["sav_file"] sav_file.seek(0) zip_file.write( - sav_file.name, '_'.join(section_name.split('/')) + '.sav') + sav_file.name, "_".join(section_name.split("/")) + ".sav" + ) # close files when we are done for (section_name, sav_def) in iteritems(sav_defs): - sav_def['sav_file'].close() + sav_def["sav_file"].close() def get_fields(self, dataview, section, key): """ Return list of element value with the key in section['elements']. """ if dataview: - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements'] - if element['title'] in dataview.columns] + \ - self.extra_columns - - return [element.get('_label_xpath') or element[key] - if self.SHOW_CHOICE_LABELS else element[key] - for element in section['elements']] + self.extra_columns + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + if element["title"] in dataview.columns + ] + self.extra_columns + + return [ + element.get("_label_xpath") or element[key] + if self.SHOW_CHOICE_LABELS + else element[key] + for element in section["elements"] + ] + self.extra_columns diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a05b32e60c..f4ba38bd7e 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -22,8 +22,8 @@ from django.shortcuts import render from django.utils import timezone from django.utils.translation import ugettext as _ -from future.moves.urllib.parse import urlparse -from future.utils import iteritems +from six.moves.urllib.parse import urlparse +from six import iteritems from json2xlsclient.client import Client from rest_framework import exceptions from savReaderWriter import SPSSIOError @@ -32,27 +32,25 @@ from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm from onadata.apps.logger.models.data_view import DataView from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.viewer.models.export import (Export, - get_export_options_query_kwargs) +from onadata.apps.viewer.models.export import Export, get_export_options_query_kwargs from onadata.apps.viewer.models.parsed_instance import query_data from onadata.libs.exceptions import J2XException, NoRecordsFoundError -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG) -from onadata.libs.utils.common_tools import (str_to_bool, - cmp_to_key, - report_exception, - retry) +from onadata.libs.utils.common_tags import DATAVIEW_EXPORT, GROUPNAME_REMOVED_FLAG +from onadata.libs.utils.common_tools import ( + str_to_bool, + cmp_to_key, + report_exception, + retry, +) from onadata.libs.utils.export_builder import ExportBuilder -from onadata.libs.utils.model_tools import (get_columns_with_hxl, - queryset_iterator) +from onadata.libs.utils.model_tools import get_columns_with_hxl, queryset_iterator from onadata.libs.utils.osm import get_combined_osm -from onadata.libs.utils.viewer_tools import (create_attachments_zipfile, - image_urls) +from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls -DEFAULT_GROUP_DELIMITER = '/' -DEFAULT_INDEX_TAGS = ('[', ']') -SUPPORTED_INDEX_TAGS = ('[', ']', '(', ')', '{', '}', '.', '_') -EXPORT_QUERY_KEY = 'query' +DEFAULT_GROUP_DELIMITER = "/" +DEFAULT_INDEX_TAGS = ("[", "]") +SUPPORTED_INDEX_TAGS = ("[", "]", "(", ")", "{", "}", ".", "_") +EXPORT_QUERY_KEY = "query" MAX_RETRIES = 3 @@ -69,8 +67,10 @@ def get_export_options(options): list of provided options to be saved with each Export object. """ export_options = { - key: value for (key, value) in iteritems(options) - if key in Export.EXPORT_OPTION_FIELDS} + key: value + for (key, value) in iteritems(options) + if key in Export.EXPORT_OPTION_FIELDS + } return export_options @@ -84,7 +84,7 @@ def get_or_create_export(export_id, xform, export_type, options): try: return Export.objects.get(pk=export_id) except Export.DoesNotExist: - if getattr(settings, 'SLAVE_DATABASES', []): + if getattr(settings, "SLAVE_DATABASES", []): from multidb.pinning import use_master with use_master: @@ -127,30 +127,32 @@ def generate_export(export_type, xform, export_id=None, options=None): start = options.get("start") export_type_func_map = { - Export.XLS_EXPORT: 'to_xls_export', - Export.CSV_EXPORT: 'to_flat_csv_export', - Export.CSV_ZIP_EXPORT: 'to_zipped_csv', - Export.SAV_ZIP_EXPORT: 'to_zipped_sav', - Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', + Export.XLS_EXPORT: "to_xls_export", + Export.CSV_EXPORT: "to_flat_csv_export", + Export.CSV_ZIP_EXPORT: "to_zipped_csv", + Export.SAV_ZIP_EXPORT: "to_zipped_sav", + Export.GOOGLE_SHEETS_EXPORT: "to_google_sheets", } if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) - records = dataview.query_data(dataview, all_data=True, - filter_query=filter_query) - total_records = dataview.query_data(dataview, - count=True)[0].get('count') + records = dataview.query_data( + dataview, all_data=True, filter_query=filter_query + ) + total_records = dataview.query_data(dataview, count=True)[0].get("count") else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: - total_records = query_data(xform, query=filter_query, start=start, - end=end, count=True)[0].get('count') + total_records = query_data( + xform, query=filter_query, start=start, end=end, count=True + )[0].get("count") else: total_records = xform.num_of_submissions @@ -158,59 +160,63 @@ def generate_export(export_type, xform, export_id=None, options=None): records = records.iterator() export_builder = ExportBuilder() - export_builder.TRUNCATE_GROUP_TITLE = True \ - if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + export_builder.TRUNCATE_GROUP_TITLE = ( + True if export_type == Export.SAV_ZIP_EXPORT else remove_group_name + ) export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) - export_builder.SPLIT_SELECT_MULTIPLES = options.get( - "split_select_multiples", True - ) + export_builder.SPLIT_SELECT_MULTIPLES = options.get("split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) - export_builder.INCLUDE_LABELS = options.get('include_labels', False) - include_reviews = options.get('include_reviews', False) - export_builder.INCLUDE_LABELS_ONLY = options.get( - 'include_labels_only', False - ) - export_builder.INCLUDE_HXL = options.get('include_hxl', False) + export_builder.INCLUDE_LABELS = options.get("include_labels", False) + include_reviews = options.get("include_reviews", False) + export_builder.INCLUDE_LABELS_ONLY = options.get("include_labels_only", False) + export_builder.INCLUDE_HXL = options.get("include_hxl", False) - export_builder.INCLUDE_IMAGES \ - = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) + export_builder.INCLUDE_IMAGES = options.get( + "include_images", settings.EXPORT_WITH_IMAGE_DEFAULT + ) - export_builder.VALUE_SELECT_MULTIPLES = options.get( - 'value_select_multiples', False) + export_builder.VALUE_SELECT_MULTIPLES = options.get("value_select_multiples", False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) - export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', - False) + export_builder.SHOW_CHOICE_LABELS = options.get("show_choice_labels", False) - export_builder.language = options.get('language') + export_builder.language = options.get("language") # 'win_excel_utf8' is only relevant for CSV exports - if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: - del options['win_excel_utf8'] + if "win_excel_utf8" in options and export_type != Export.CSV_EXPORT: + del options["win_excel_utf8"] export_builder.INCLUDE_REVIEWS = include_reviews - export_builder.set_survey(xform.survey, xform, - include_reviews=include_reviews) + export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( - xform.survey_elements) + xform.survey_elements + ) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( - temp_file.name, records, username, id_string, filter_query, - start=start, end=end, dataview=dataview, xform=xform, - options=options, columns_with_hxl=columns_with_hxl, - total_records=total_records + temp_file.name, + records, + username, + id_string, + filter_query, + start=start, + end=end, + dataview=dataview, + xform=xform, + options=options, + columns_with_hxl=columns_with_hxl, + total_records=total_records, ) except NoRecordsFoundError: pass @@ -223,8 +229,7 @@ def generate_export(export_type, xform, export_id=None, options=None): return export # generate filename - basename = "%s_%s" % ( - id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename @@ -238,17 +243,11 @@ def generate_export(export_type, xform, export_id=None, options=None): while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) - export_filename = default_storage.save(file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) @@ -275,20 +274,25 @@ def create_export_object(xform, export_type, options): Return an export object that has not been saved to the database. """ export_options = get_export_options(options) - return Export(xform=xform, export_type=export_type, options=export_options, - created_on=timezone.now()) + return Export( + xform=xform, + export_type=export_type, + options=export_options, + created_on=timezone.now(), + ) -def check_pending_export(xform, export_type, options, - minutes=getattr(settings, 'PENDING_EXPORT_TIME', 5)): +def check_pending_export( + xform, export_type, options, minutes=getattr(settings, "PENDING_EXPORT_TIME", 5) +): """ - Check for pending export done within a specific period of time and - returns the export - :param xform: - :param export_type: - :param options: - :param minutes - :return: + Check for pending export done within a specific period of time and + returns the export + :param xform: + :param export_type: + :param options: + :param minutes + :return: """ created_time = timezone.now() - timedelta(minutes=minutes) export_options_kwargs = get_export_options_query_kwargs(options) @@ -303,10 +307,7 @@ def check_pending_export(xform, export_type, options, return export -def should_create_new_export(xform, - export_type, - options, - request=None): +def should_create_new_export(xform, export_type, options, request=None): """ Function that determines whether to create a new export. param: xform @@ -319,27 +320,27 @@ def should_create_new_export(xform, index_tag: ('[', ']') or ('_', '_') params: request: Get params are used to determine if new export is required """ - split_select_multiples = options.get('split_select_multiples', True) + split_select_multiples = options.get("split_select_multiples", True) - if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False): + if getattr(settings, "SHOULD_ALWAYS_CREATE_NEW_EXPORT", False): return True - if (request and (frozenset(list(request.GET)) & - frozenset(['start', 'end', 'data_id']))) or\ - not split_select_multiples: + if ( + request + and (frozenset(list(request.GET)) & frozenset(["start", "end", "data_id"])) + ) or not split_select_multiples: return True export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) if options.get(EXPORT_QUERY_KEY) is None: export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY) - if export_query.count() == 0 or\ - Export.exports_outdated(xform, export_type, options=options): + if export_query.count() == 0 or Export.exports_outdated( + xform, export_type, options=options + ): return True return False @@ -361,12 +362,10 @@ def newest_export_for(xform, export_type, options): export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( - xform=xform, - export_type=export_type, - **export_options_kwargs + xform=xform, export_type=export_type, **export_options_kwargs ) - return export_query.latest('created_on') + return export_query.latest("created_on") def increment_index_in_filename(filename): @@ -390,9 +389,9 @@ def increment_index_in_filename(filename): # pylint: disable=R0913 -def generate_attachments_zip_export(export_type, username, id_string, - export_id=None, options=None, - xform=None): +def generate_attachments_zip_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates zip export of attachments. @@ -413,45 +412,44 @@ def generate_attachments_zip_export(export_type, username, id_string, dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ - rec.get('_id') + rec.get("_id") for rec in dataview.query_data( - dataview, all_data=True, filter_query=filter_query)], - instance__deleted_at__isnull=True) + dataview, all_data=True, filter_query=filter_query + ) + ], + instance__deleted_at__isnull=True, + ) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) - attachments = Attachment.objects.filter( - instance__deleted_at__isnull=True) + attachments = Attachment.objects.filter(instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( instance__xform_id__in=[ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', flat=True)]).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] + ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) else: - attachments = attachments.filter( - instance__xform_id=xform.pk).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) - - filename = "%s_%s.%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), - export_type.lower()) - file_path = os.path.join( - username, - 'exports', + attachments = attachments.filter(instance__xform_id=xform.pk).filter( + instance_id__in=[i_id["_id"] for i_id in instance_ids] + ) + + filename = "%s_%s.%s" % ( id_string, - export_type, - filename) + datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), + export_type.lower(), + ) + file_path = os.path.join(username, "exports", id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: - temp_file = builtins.open(zip_file.name, 'rb') - filename = default_storage.save( - file_path, - File(temp_file, file_path)) + temp_file = builtins.open(zip_file.name, "rb") + filename = default_storage.save(file_path, File(temp_file, file_path)) finally: temp_file.close() finally: @@ -467,7 +465,7 @@ def generate_attachments_zip_export(export_type, username, id_string, def write_temp_file_to_path(suffix, content, file_path): - """ Write a temp file and return the name of the file. + """Write a temp file and return the name of the file. :param suffix: The file suffix :param content: The content to write :param file_path: The path to write the temp file to @@ -476,16 +474,14 @@ def write_temp_file_to_path(suffix, content, file_path): temp_file = NamedTemporaryFile(suffix=suffix) temp_file.write(content) temp_file.seek(0) - export_filename = default_storage.save( - file_path, - File(temp_file, file_path)) + export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() return export_filename def get_or_create_export_object(export_id, options, xform, export_type): - """ Get or create export object. + """Get or create export object. :param export_id: Export ID :param options: Options to convert to export options @@ -497,16 +493,17 @@ def get_or_create_export_object(export_id, options, xform, export_type): export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) return export # pylint: disable=R0913 -def generate_kml_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_kml_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates kml export for geographical data @@ -524,25 +521,18 @@ def generate_kml_export(export_type, username, id_string, export_id=None, xform = XForm.objects.get(user__username=username, id_string=id_string) response = render( - None, 'survey.kml', - {'data': kml_export_data(id_string, user, xform=xform)} + None, "survey.kml", {"data": kml_export_data(id_string, user, xform=xform)} ) - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + export_type.lower() - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path( - export_type.lower(), response.content, file_path) + export_type.lower(), response.content, file_path + ) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) export.filedir, export.filename = os.path.split(export_filename) export.internal_status = Export.SUCCESSFUL @@ -555,6 +545,7 @@ def kml_export_data(id_string, user, xform=None): """ KML export data from form submissions. """ + def cached_get_labels(xpath): """ Get and Cache labels for the XForm. @@ -567,16 +558,21 @@ def cached_get_labels(xpath): xform = xform or XForm.objects.get(id_string=id_string, user=user) - data_kwargs = {'geom__isnull': False} + data_kwargs = {"geom__isnull": False} if xform.is_merged_dataset: - data_kwargs.update({ - 'xform_id__in': - [i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] - }) + data_kwargs.update( + { + "xform_id__in": [ + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] + } + ) else: - data_kwargs.update({'xform_id': xform.pk}) - instances = Instance.objects.filter(**data_kwargs).order_by('id') + data_kwargs.update({"xform_id": xform.pk}) + instances = Instance.objects.filter(**data_kwargs).order_by("id") data_for_template = [] labels = {} for instance in queryset_iterator(instances): @@ -585,22 +581,26 @@ def cached_get_labels(xpath): xpaths = list(data_for_display) xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp())) table_rows = [ - '%s%s' % - (cached_get_labels(xpath), data_for_display[xpath]) for xpath in - xpaths if not xpath.startswith(u"_")] + "%s%s" + % (cached_get_labels(xpath), data_for_display[xpath]) + for xpath in xpaths + if not xpath.startswith("_") + ] img_urls = image_urls(instance) if instance.point: - data_for_template.append({ - 'name': instance.xform.id_string, - 'id': instance.id, - 'lat': instance.point.y, - 'lng': instance.point.x, - 'image_urls': img_urls, - 'table': '%s' - '
    ' % (img_urls[0] if img_urls else "", - ''.join(table_rows))}) + data_for_template.append( + { + "name": instance.xform.id_string, + "id": instance.id, + "lat": instance.point.y, + "lng": instance.point.x, + "image_urls": img_urls, + "table": '%s' + "
    " % (img_urls[0] if img_urls else "", "".join(table_rows)), + } + ) return data_for_template @@ -608,20 +608,24 @@ def cached_get_labels(xpath): def get_osm_data_kwargs(xform): """Return kwargs for OsmData queryset for given xform""" - kwargs = {'instance__deleted_at__isnull': True} + kwargs = {"instance__deleted_at__isnull": True} if xform.is_merged_dataset: - kwargs['instance__xform_id__in'] = [ - i for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True)] + kwargs["instance__xform_id__in"] = [ + i + for i in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) + ] else: - kwargs['instance__xform_id'] = xform.pk + kwargs["instance__xform_id"] = xform.pk return kwargs -def generate_osm_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_osm_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates osm export for OpenStreetMap data @@ -642,20 +646,13 @@ def generate_osm_export(export_type, username, id_string, export_id=None, osm_list = OsmData.objects.filter(**kwargs) content = get_combined_osm(osm_list) - basename = "%s_%s" % (id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension - file_path = os.path.join( - username, - 'exports', - id_string, - export_type, - filename) + file_path = os.path.join(username, "exports", id_string, export_type, filename) export_filename = write_temp_file_to_path(extension, content, file_path) - export = get_or_create_export_object( - export_id, options, xform, export_type) + export = get_or_create_export_object(export_id, options, xform, export_type) dir_name, basename = os.path.split(export_filename) export.filedir = dir_name @@ -667,8 +664,7 @@ def generate_osm_export(export_type, username, id_string, export_id=None, def _get_records(instances): - return [clean_keys_of_slashes(instance) - for instance in instances] + return [clean_keys_of_slashes(instance) for instance in instances] def clean_keys_of_slashes(record): @@ -679,10 +675,9 @@ def clean_keys_of_slashes(record): """ for key in list(record): value = record[key] - if '/' in key: + if "/" in key: # replace with _ - record[key.replace('/', '_')]\ - = record.pop(key) + record[key.replace("/", "_")] = record.pop(key) # Check if the value is a list containing nested dict and apply same if value: if isinstance(value, list) and isinstance(value[0], dict): @@ -698,6 +693,7 @@ def _get_server_from_metadata(xform, meta, token): report_templates = MetaData.external_export(xform) except MetaData.DoesNotExist: from multidb.pinning import use_master + with use_master: report_templates = MetaData.external_export(xform) @@ -705,7 +701,7 @@ def _get_server_from_metadata(xform, meta, token): try: int(meta) except ValueError: - raise Exception(u"Invalid metadata pk {0}".format(meta)) + raise Exception("Invalid metadata pk {0}".format(meta)) # Get the external server from the metadata result = report_templates.get(pk=meta) @@ -718,7 +714,8 @@ def _get_server_from_metadata(xform, meta, token): # Take the latest value in the metadata if not report_templates: raise Exception( - u"Could not find the template token: Please upload template.") + "Could not find the template token: Please upload template." + ) server = report_templates[0].external_export_url name = report_templates[0].external_export_name @@ -726,8 +723,9 @@ def _get_server_from_metadata(xform, meta, token): return server, name -def generate_external_export(export_type, username, id_string, export_id=None, - options=None, xform=None): +def generate_external_export( + export_type, username, id_string, export_id=None, options=None, xform=None +): """ Generates external export using ONA data through an external service. @@ -748,7 +746,8 @@ def generate_external_export(export_type, username, id_string, export_id=None, if xform is None: xform = XForm.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) @@ -758,14 +757,13 @@ def generate_external_export(export_type, username, id_string, export_id=None, token = parsed_url.path[5:] - ser = parsed_url.scheme + '://' + parsed_url.netloc + ser = parsed_url.scheme + "://" + parsed_url.netloc # Get single submission data if data_id: - inst = Instance.objects.filter(xform__user=user, - xform__id_string=id_string, - deleted_at=None, - pk=data_id) + inst = Instance.objects.filter( + xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id + ) instances = [inst[0].json if inst else {}] else: @@ -780,20 +778,18 @@ def generate_external_export(export_type, username, id_string, export_id=None, client = Client(ser) response = client.xls.create(token, json.dumps(records)) - if hasattr(client.xls.conn, 'last_response'): + if hasattr(client.xls.conn, "last_response"): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( - u"J2X client could not generate report. Server -> {0}," - u" Error-> {1}".format(server, e) + "J2X client could not generate report. Server -> {0}," + " Error-> {1}".format(server, e) ) else: if not server: - raise J2XException(u"External server not set") + raise J2XException("External server not set") elif not records: - raise J2XException( - u"No record to export. Form -> {0}".format(id_string) - ) + raise J2XException("No record to export. Form -> {0}".format(id_string)) # get or create export object if export_id: @@ -804,14 +800,14 @@ def generate_external_export(export_type, username, id_string, export_id=None, export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) - export = Export.objects.create(xform=xform, - export_type=export_type, - options=export_options) + export = Export.objects.create( + xform=xform, export_type=export_type, options=export_options + ) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL - export.filename = name + '-' + response[5:] if name else response[5:] + export.filename = name + "-" + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED @@ -832,10 +828,10 @@ def upload_template_for_external_export(server, file_obj): response = client.template.create(template_file=file_obj) status_code = None - if hasattr(client.template.conn, 'last_response'): + if hasattr(client.template.conn, "last_response"): status_code = client.template.conn.last_response.status_code - return str(status_code) + '|' + response + return str(status_code) + "|" + response def parse_request_export_options(params): # pylint: disable=too-many-branches @@ -845,71 +841,73 @@ def parse_request_export_options(params): # pylint: disable=too-many-branches removed, the group delimiter, and a boolean for whether select multiples should be split. """ - boolean_list = ['true', 'false'] + boolean_list = ["true", "false"] options = {} - remove_group_name = params.get('remove_group_name') and \ - params.get('remove_group_name').lower() - binary_select_multiples = params.get('binary_select_multiples') and \ - params.get('binary_select_multiples').lower() - do_not_split_select_multiples = params.get( - 'do_not_split_select_multiples') - include_labels = params.get('include_labels', False) - include_reviews = params.get('include_reviews', False) - include_labels_only = params.get('include_labels_only', False) - include_hxl = params.get('include_hxl', True) - value_select_multiples = params.get('value_select_multiples') and \ - params.get('value_select_multiples').lower() - show_choice_labels = params.get('show_choice_labels') and \ - params.get('show_choice_labels').lower() + remove_group_name = ( + params.get("remove_group_name") and params.get("remove_group_name").lower() + ) + binary_select_multiples = ( + params.get("binary_select_multiples") + and params.get("binary_select_multiples").lower() + ) + do_not_split_select_multiples = params.get("do_not_split_select_multiples") + include_labels = params.get("include_labels", False) + include_reviews = params.get("include_reviews", False) + include_labels_only = params.get("include_labels_only", False) + include_hxl = params.get("include_hxl", True) + value_select_multiples = ( + params.get("value_select_multiples") + and params.get("value_select_multiples").lower() + ) + show_choice_labels = ( + params.get("show_choice_labels") and params.get("show_choice_labels").lower() + ) if include_labels is not None: - options['include_labels'] = str_to_bool(include_labels) + options["include_labels"] = str_to_bool(include_labels) if include_reviews is not None: - options['include_reviews'] = str_to_bool(include_reviews) + options["include_reviews"] = str_to_bool(include_reviews) if include_labels_only is not None: - options['include_labels_only'] = str_to_bool(include_labels_only) + options["include_labels_only"] = str_to_bool(include_labels_only) if include_hxl is not None: - options['include_hxl'] = str_to_bool(include_hxl) + options["include_hxl"] = str_to_bool(include_hxl) if remove_group_name in boolean_list: options["remove_group_name"] = str_to_bool(remove_group_name) else: options["remove_group_name"] = False - if params.get("group_delimiter") in ['.', DEFAULT_GROUP_DELIMITER]: - options['group_delimiter'] = params.get("group_delimiter") + if params.get("group_delimiter") in [".", DEFAULT_GROUP_DELIMITER]: + options["group_delimiter"] = params.get("group_delimiter") else: - options['group_delimiter'] = DEFAULT_GROUP_DELIMITER + options["group_delimiter"] = DEFAULT_GROUP_DELIMITER - options['split_select_multiples'] = \ - not str_to_bool(do_not_split_select_multiples) + options["split_select_multiples"] = not str_to_bool(do_not_split_select_multiples) if binary_select_multiples and binary_select_multiples in boolean_list: - options['binary_select_multiples'] = str_to_bool( - binary_select_multiples) + options["binary_select_multiples"] = str_to_bool(binary_select_multiples) - if 'include_images' in params: - options["include_images"] = str_to_bool( - params.get("include_images")) + if "include_images" in params: + options["include_images"] = str_to_bool(params.get("include_images")) else: options["include_images"] = settings.EXPORT_WITH_IMAGE_DEFAULT - options['win_excel_utf8'] = str_to_bool(params.get('win_excel_utf8')) + options["win_excel_utf8"] = str_to_bool(params.get("win_excel_utf8")) if value_select_multiples and value_select_multiples in boolean_list: - options['value_select_multiples'] = str_to_bool(value_select_multiples) + options["value_select_multiples"] = str_to_bool(value_select_multiples) if show_choice_labels and show_choice_labels in boolean_list: - options['show_choice_labels'] = str_to_bool(show_choice_labels) + options["show_choice_labels"] = str_to_bool(show_choice_labels) index_tags = get_repeat_index_tags(params.get("repeat_index_tags")) if index_tags: - options['repeat_index_tags'] = index_tags + options["repeat_index_tags"] = index_tags - if 'language' in params: - options['language'] = params.get('language') + if "language" in params: + options["language"] = params.get("language") return options @@ -921,7 +919,7 @@ def get_repeat_index_tags(index_tags): Retuns a tuple of two strings with SUPPORTED_INDEX_TAGS, """ if isinstance(index_tags, six.string_types): - index_tags = tuple(index_tags.split(',')) + index_tags = tuple(index_tags.split(",")) length = len(index_tags) if length == 1: index_tags = (index_tags[0], index_tags[0]) @@ -932,8 +930,11 @@ def get_repeat_index_tags(index_tags): for tag in index_tags: if tag not in SUPPORTED_INDEX_TAGS: - raise exceptions.ParseError(_( - "The tag %s is not supported, supported tags are %s" % - (tag, SUPPORTED_INDEX_TAGS))) + raise exceptions.ParseError( + _( + "The tag %s is not supported, supported tags are %s" + % (tag, SUPPORTED_INDEX_TAGS) + ) + ) return index_tags diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index afe3b0a18e..bae078432d 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,6 +1,6 @@ import hashlib -from future.moves.urllib.parse import urlencode -from future.moves.urllib.request import urlopen +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen DEFAULT_GRAVATAR = "https://ona.io/static/images/default_avatar.png" GRAVATAR_ENDPOINT = "https://secure.gravatar.com/avatar/" @@ -8,12 +8,16 @@ def email_md5(user): - return hashlib.md5(user.email.lower().encode('utf-8')).hexdigest() + return hashlib.md5(user.email.lower().encode("utf-8")).hexdigest() def get_gravatar_img_link(user): - return GRAVATAR_ENDPOINT + email_md5(user) + "?" + urlencode({ - 'd': DEFAULT_GRAVATAR, 's': str(GRAVATAR_SIZE)}) + return ( + GRAVATAR_ENDPOINT + + email_md5(user) + + "?" + + urlencode({"d": DEFAULT_GRAVATAR, "s": str(GRAVATAR_SIZE)}) + ) def gravatar_exists(user): diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 23393476c1..f4a1c17839 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -6,11 +6,10 @@ import logging -from django.contrib.gis.geos import (GeometryCollection, LineString, Point, - Polygon) +from django.contrib.gis.geos import GeometryCollection, LineString, Point, Polygon from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from future.utils import iteritems +from six import iteritems from lxml import etree from onadata.apps.logger.models.attachment import Attachment @@ -26,8 +25,8 @@ def _get_xml_obj(xml): try: return etree.fromstring(xml) # pylint: disable=E1101 except etree.XMLSyntaxError as e: # pylint: disable=E1101 - if 'Attribute action redefined' in e.msg: - xml = xml.replace(b'action="modify" ', b'') + if "Attribute action redefined" in e.msg: + xml = xml.replace(b'action="modify" ', b"") return _get_xml_obj(xml) @@ -37,7 +36,7 @@ def _get_node(ref, root): nodes = root.xpath('//node[@id="{}"]'.format(ref)) if nodes: node = nodes[0] - point = Point(float(node.get('lon')), float(node.get('lat'))) + point = Point(float(node.get("lon")), float(node.get("lat"))) return point @@ -46,9 +45,10 @@ def get_combined_osm(osm_list): """ Combine osm xml form list of OsmData objects """ - xml = '' - if (osm_list and isinstance(osm_list, list)) \ - or isinstance(osm_list, models.QuerySet): + xml = "" + if (osm_list and isinstance(osm_list, list)) or isinstance( + osm_list, models.QuerySet + ): osm = None for osm_data in osm_list: osm_xml = osm_data.xml @@ -62,65 +62,59 @@ def get_combined_osm(osm_list): osm.append(child) if osm is not None: # pylint: disable=E1101 - return etree.tostring(osm, encoding='utf-8', xml_declaration=True) + return etree.tostring(osm, encoding="utf-8", xml_declaration=True) elif isinstance(osm_list, dict): - if 'detail' in osm_list: - xml = '%s' % osm_list['detail'] - return xml.encode('utf-8') + if "detail" in osm_list: + xml = "%s" % osm_list["detail"] + return xml.encode("utf-8") def parse_osm_ways(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for way in root.findall('way'): + for way in root.findall("way"): geom = None points = [] - for node in way.findall('nd'): - points.append(_get_node(node.get('ref'), root)) + for node in way.findall("nd"): + points.append(_get_node(node.get("ref"), root)) try: geom = Polygon(points) except GEOSException: geom = LineString(points) tags = parse_osm_tags(way, include_osm_id) - items.append({ - 'osm_id': way.get('id'), - 'geom': geom, - 'tags': tags, - 'osm_type': 'way' - }) + items.append( + {"osm_id": way.get("id"), "geom": geom, "tags": tags, "osm_type": "way"} + ) return items def parse_osm_nodes(osm_xml, include_osm_id=False): - """Converts an OSM XMl to a list of GEOSGeometry objects """ + """Converts an OSM XMl to a list of GEOSGeometry objects""" items = [] root = _get_xml_obj(osm_xml) - for node in root.findall('node'): - point = Point(float(node.get('lon')), float(node.get('lat'))) + for node in root.findall("node"): + point = Point(float(node.get("lon")), float(node.get("lat"))) tags = parse_osm_tags(node, include_osm_id) - items.append({ - 'osm_id': node.get('id'), - 'geom': point, - 'tags': tags, - 'osm_type': 'node' - }) + items.append( + {"osm_id": node.get("id"), "geom": point, "tags": tags, "osm_type": "node"} + ) return items def parse_osm_tags(node, include_osm_id=False): """Retrieves all the tags from a osm xml node""" - tags = {} if not include_osm_id else {node.tag + ':id': node.get('id')} - for tag in node.findall('tag'): - key, val = tag.attrib['k'], tag.attrib['v'] - if val == '' or val.upper() == 'FIXME': + tags = {} if not include_osm_id else {node.tag + ":id": node.get("id")} + for tag in node.findall("tag"): + key, val = tag.attrib["k"], tag.attrib["v"] + if val == "" or val.upper() == "FIXME": continue tags.update({key: val}) @@ -153,24 +147,24 @@ def save_osm_data(instance_id): Includes the OSM data in the specified submission json data. """ instance = Instance.objects.filter(pk=instance_id).first() - osm_attachments = instance.attachments.filter(extension=Attachment.OSM) \ - if instance else None + osm_attachments = ( + instance.attachments.filter(extension=Attachment.OSM) if instance else None + ) if instance and osm_attachments: fields = [ f.get_abbreviated_xpath() - for f in instance.xform.get_survey_elements_of_type('osm') + for f in instance.xform.get_survey_elements_of_type("osm") ] osm_filenames = { - field: instance.json[field] - for field in fields if field in instance.json + field: instance.json[field] for field in fields if field in instance.json } for osm in osm_attachments: try: osm_xml = osm.media_file.read() if isinstance(osm_xml, bytes): - osm_xml = osm_xml.decode('utf-8') + osm_xml = osm_xml.decode("utf-8") except IOError as e: logging.exception("IOError saving osm data: %s" % str(e)) continue @@ -178,7 +172,7 @@ def save_osm_data(instance_id): filename = None field_name = None for k, v in osm_filenames.items(): - if osm.filename.startswith(v.replace('.osm', '')): + if osm.filename.startswith(v.replace(".osm", "")): filename = v field_name = k break @@ -193,26 +187,27 @@ def save_osm_data(instance_id): osm_data = OsmData( instance=instance, xml=osm_xml, - osm_id=osmd['osm_id'], - osm_type=osmd['osm_type'], - tags=osmd['tags'], - geom=GeometryCollection(osmd['geom']), + osm_id=osmd["osm_id"], + osm_type=osmd["osm_type"], + tags=osmd["tags"], + geom=GeometryCollection(osmd["geom"]), filename=filename, - field_name=field_name) + field_name=field_name, + ) osm_data.save() except IntegrityError: with transaction.atomic(): - osm_data = OsmData.objects.exclude( - xml=osm_xml).filter( - instance=instance, - field_name=field_name).first() + osm_data = ( + OsmData.objects.exclude(xml=osm_xml) + .filter(instance=instance, field_name=field_name) + .first() + ) if osm_data: osm_data.xml = osm_xml - osm_data.osm_id = osmd['osm_id'] - osm_data.osm_type = osmd['osm_type'] - osm_data.tags = osmd['tags'] - osm_data.geom = GeometryCollection( - osmd['geom']) + osm_data.osm_id = osmd["osm_id"] + osm_data.osm_type = osmd["osm_type"] + osm_data.tags = osmd["tags"] + osm_data.geom = GeometryCollection(osmd["geom"]) osm_data.filename = filename osm_data.save() instance.save() diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 62b03ff00b..05a087b28b 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -593,3 +593,5 @@ def configure_logging(logger, **kwargs): XFORM_SUBMISSION_STAT_CACHE_TIME = 600 XFORM_CHARTS_CACHE_TIME = 600 + +SLAVE_DATABASES = [] diff --git a/requirements/base.in b/requirements/base.in index ee5efd28bc..703d532072 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -3,10 +3,10 @@ # installed from Git -e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform -#-e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest -#-e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest -#-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router -#-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip -#-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient -#-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client -#-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest +-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router +-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip +-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient +-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client +-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index 9a694656b8..4604b22f8c 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -5,6 +5,20 @@ # pip-compile --output-file=requirements/base.pip requirements/base.in # -e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform +-e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest + # via -r requirements/base.in +-e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router + # via -r requirements/base.in +-e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client + # via -r requirements/base.in +-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc + # via -r requirements/base.in +-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip + # via -r requirements/base.in +-e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest + # via -r requirements/base.in +-e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient + # via -r requirements/base.in alabaster==0.7.12 # via sphinx amqp==5.1.1 @@ -17,18 +31,34 @@ asgiref==3.5.0 # via django async-timeout==4.0.2 # via redis +attrs==21.4.0 + # via + # jsonlines + # jsonschema babel==2.9.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery +boto3==1.21.43 + # via tabulator +botocore==1.24.43 + # via + # boto3 + # s3transfer +cached-property==1.5.2 + # via tableschema celery==5.2.6 # via onadata certifi==2021.10.8 # via requests cffi==1.15.0 # via cryptography +chardet==4.0.0 + # via + # datapackage + # tabulator charset-normalizer==2.0.12 # via requests click==8.1.2 @@ -37,6 +67,9 @@ click==8.1.2 # click-didyoumean # click-plugins # click-repl + # datapackage + # tableschema + # tabulator click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 @@ -47,6 +80,9 @@ cryptography==36.0.2 # via # jwcrypto # onadata + # pyjwt +datapackage==1.15.2 + # via pyfloip defusedxml==0.7.1 # via # djangorestframework-xml @@ -73,6 +109,7 @@ django==3.2.13 # djangorestframework # djangorestframework-guardian # djangorestframework-jsonapi + # ona-oidc # onadata django-activity-stream==1.4.0 # via onadata @@ -112,6 +149,7 @@ djangorestframework==3.13.1 # djangorestframework-gis # djangorestframework-guardian # djangorestframework-jsonapi + # ona-oidc # onadata djangorestframework-csv==2.1.1 # via onadata @@ -137,26 +175,50 @@ flake8==4.0.1 # via onadata fleming==0.7.0 # via django-query-builder +future==0.18.2 + # via python-json2xlsclient geojson==2.5.0 # via onadata +greenlet==1.1.2 + # via sqlalchemy httmock==1.4.0 # via onadata httplib2==0.20.4 - # via onadata + # via + # oauth2client + # onadata idna==3.3 # via requests +ijson==3.1.4 + # via tabulator imagesize==1.3.0 # via sphinx inflection==0.5.1 # via djangorestframework-jsonapi +isodate==0.6.1 + # via tableschema jinja2==3.1.1 # via sphinx +jmespath==1.0.0 + # via + # boto3 + # botocore +jsonlines==3.0.0 + # via tabulator jsonpickle==2.1.0 # via onadata +jsonpointer==2.3 + # via datapackage +jsonschema==4.4.0 + # via + # datapackage + # tableschema jwcrypto==1.0 # via django-oauth-toolkit kombu==5.2.4 # via celery +linear-tsv==1.1.0 + # via tabulator lxml==4.8.0 # via onadata markdown==3.3.6 @@ -181,6 +243,7 @@ openpyxl==3.0.9 # via # onadata # pyxform + # tabulator packaging==21.3 # via # redis @@ -193,6 +256,15 @@ pillow==9.1.0 # onadata prompt-toolkit==3.0.29 # via click-repl +psycopg2-binary==2.9.3 + # via onadata +pyasn1==0.4.8 + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via oauth2client pycodestyle==2.8.0 # via flake8 pycparser==2.21 @@ -201,8 +273,10 @@ pyflakes==2.4.0 # via flake8 pygments==2.11.2 # via sphinx -pyjwt==2.3.0 - # via onadata +pyjwt[crypto]==2.3.0 + # via + # ona-oidc + # onadata pylibmc==1.6.1 # via onadata pymongo==4.1.1 @@ -211,11 +285,15 @@ pyparsing==3.0.8 # via # httplib2 # packaging +pyrsistent==0.18.1 + # via jsonschema python-dateutil==2.8.2 # via # analytics-python + # botocore # fleming # onadata + # tableschema python-memcached==1.59 # via onadata pytz==2022.1 @@ -236,13 +314,24 @@ redis==4.2.2 requests==2.27.1 # via # analytics-python + # datapackage # django-oauth-toolkit # httmock + # ona-oidc # onadata + # python-json2xlsclient # requests-mock # sphinx + # tableschema + # tabulator requests-mock==1.9.3 # via onadata +rfc3986==2.0.0 + # via tableschema +rsa==4.8 + # via oauth2client +s3transfer==0.5.2 + # via boto3 savreaderwriter==3.4.2 # via onadata simplejson==3.17.6 @@ -252,11 +341,17 @@ six==1.16.0 # analytics-python # appoptics-metrics # click-repl + # datapackage # django-query-builder # djangorestframework-csv + # isodate + # linear-tsv + # oauth2client # python-dateutil # python-memcached # requests-mock + # tableschema + # tabulator snowballstemmer==2.2.0 # via sphinx sphinx==4.5.0 @@ -273,16 +368,29 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +sqlalchemy==1.4.35 + # via tabulator sqlparse==0.4.2 # via # django # django-debug-toolbar +tableschema==1.20.2 + # via datapackage +tabulator==1.53.5 + # via + # datapackage + # tableschema unicodecsv==0.14.1 # via + # datapackage # djangorestframework-csv # onadata + # tableschema + # tabulator urllib3==1.26.9 - # via requests + # via + # botocore + # requests uwsgi==2.0.20 # via onadata vine==5.0.0 @@ -298,6 +406,7 @@ xlrd==2.0.1 # via # onadata # pyxform + # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 diff --git a/setup.cfg b/setup.cfg index 1b794f9ed5..398ac0baa8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,7 +55,7 @@ install_requires = #tagging django-taggit #database - #psycopg2-binary>2.7.1 + psycopg2-binary>2.7.1 pymongo #sms support dict2xml From 610c836b247cbbe71598f9792fff1c2e3757fc6b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:22:15 +0300 Subject: [PATCH 011/234] django.utils.six - Remove usage of this vendored library or switch to six. --- .../commands/test_create_user_profiles.py | 17 +- .../management/commands/test_delete_users.py | 32 +- .../test_increase_odk_token_lifetime.py | 14 +- .../test_retrieve_org_or_project_list.py | 80 ++- .../apps/api/viewsets/briefcase_viewset.py | 172 +++--- onadata/apps/api/viewsets/data_viewset.py | 452 ++++++++------- onadata/apps/api/viewsets/xform_viewset.py | 4 +- .../tests/test_transfer_project_command.py | 129 +++-- onadata/apps/logger/views.py | 528 ++++++++++-------- onadata/libs/filters.py | 370 ++++++------ onadata/libs/renderers/renderers.py | 4 +- onadata/libs/utils/api_export_tools.py | 367 ++++++------ onadata/settings/common.py | 6 +- 13 files changed, 1143 insertions(+), 1032 deletions(-) diff --git a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py index 595d2226e0..e657b05999 100644 --- a/onadata/apps/api/tests/management/commands/test_create_user_profiles.py +++ b/onadata/apps/api/tests/management/commands/test_create_user_profiles.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """Test create user profile management command.""" from django.contrib.auth.models import User -from onadata.apps.main.models.user_profile import UserProfile from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO + +from onadata.apps.main.models.user_profile import UserProfile from onadata.apps.main.tests.test_base import TestBase @@ -16,19 +17,13 @@ def test_create_user_profiles(self): successfully creates a user profile for users missing profiles. """ - user = User.objects.create( - username='dave', email='dave@example.com') + user = User.objects.create(username="dave", email="dave@example.com") with self.assertRaises(UserProfile.DoesNotExist): _ = user.profile out = StringIO() - call_command( - 'create_user_profiles', - stdout=out - ) + call_command("create_user_profiles", stdout=out) user.refresh_from_db() # Assert profile is retrievable; profile = user.profile self.assertEqual(profile.user, user) - self.assertEqual( - 'User Profiles successfully created.\n', - out.getvalue()) + self.assertEqual("User Profiles successfully created.\n", out.getvalue()) diff --git a/onadata/apps/api/tests/management/commands/test_delete_users.py b/onadata/apps/api/tests/management/commands/test_delete_users.py index 329e7eac89..ce975e13f5 100644 --- a/onadata/apps/api/tests/management/commands/test_delete_users.py +++ b/onadata/apps/api/tests/management/commands/test_delete_users.py @@ -3,59 +3,51 @@ """ import sys from unittest import mock -from django.utils.six import StringIO +from six import StringIO from django.contrib.auth.models import User from django.core.management import call_command from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.api.management.commands.delete_users import \ - get_user_object_stats +from onadata.apps.api.management.commands.delete_users import get_user_object_stats class DeleteUserTest(TestBase): """ Test delete user management command. """ + def test_delete_users_with_input(self): """ Test that a user account is deleted automatically when the user_input field is provided as true """ - user = User.objects.create( - username="bruce", - email="bruce@gmail.com") + user = User.objects.create(username="bruce", email="bruce@gmail.com") username = user.username email = user.email out = StringIO() sys.stdout = out - new_user_details = [username+':'+email] + new_user_details = [username + ":" + email] call_command( - 'delete_users', - user_details=new_user_details, - user_input='True', - stdout=out) + "delete_users", user_details=new_user_details, user_input="True", stdout=out + ) - self.assertEqual( - "User bruce deleted successfully.", - out.getvalue()) + self.assertEqual("User bruce deleted successfully.", out.getvalue()) with self.assertRaises(User.DoesNotExist): User.objects.get(email="bruce@gmail.com") - @mock.patch( - "onadata.apps.api.management.commands.delete_users.input") + @mock.patch("onadata.apps.api.management.commands.delete_users.input") def test_delete_users_no_input(self, mock_input): # pylint: disable=R0201 """ Test that when user_input is not provided, the user account stats are provided for that user account before deletion """ - user = User.objects.create( - username="barbie", - email="barbie@gmail.com") + user = User.objects.create(username="barbie", email="barbie@gmail.com") username = user.username get_user_object_stats(username) mock_input.assert_called_with( "User account 'barbie' has 0 projects, " "0 forms and 0 submissions. " - "Do you wish to continue deleting this account?") + "Do you wish to continue deleting this account?" + ) diff --git a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py index 3cd040d68a..af54161d35 100644 --- a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py +++ b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.models.odk_token import ODKToken @@ -10,21 +10,17 @@ class IncreaseODKTokenLifetimeTest(TestBase): def test_increase_odk_token_lifetime(self): - user = User.objects.create( - username='dave', email='dave@example.com') + user = User.objects.create(username="dave", email="dave@example.com") token = ODKToken.objects.create(user=user) expiry_date = token.expires out = StringIO() call_command( - 'increase_odk_token_lifetime', - days=2, - username=user.username, - stdout=out + "increase_odk_token_lifetime", days=2, username=user.username, stdout=out ) self.assertEqual( - 'Increased the lifetime of ODK Token for user dave\n', - out.getvalue()) + "Increased the lifetime of ODK Token for user dave\n", out.getvalue() + ) token.refresh_from_db() self.assertEqual(expiry_date + timedelta(days=2), token.expires) diff --git a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py index 1fe1a383e7..703f838dfe 100644 --- a/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py +++ b/onadata/apps/api/tests/management/commands/test_retrieve_org_or_project_list.py @@ -1,11 +1,10 @@ import json + from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.libs.serializers.share_project_serializer import \ - ShareProjectSerializer +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.libs.serializers.share_project_serializer import ShareProjectSerializer class TestRetrieveOrgOrProjectListCommand(TestAbstractViewSet): @@ -13,52 +12,48 @@ def test_retrieve_org_or_project_list(self): self._org_create() self._project_create() - user = self._create_user_profile( - {'username': 'alice'}).user + user = self._create_user_profile({"username": "alice"}).user share_data = { - 'project': self.project.id, - 'username': user.username, - 'role': 'editor' + "project": self.project.id, + "username": user.username, + "role": "editor", } serializer = ShareProjectSerializer(data=share_data) self.assertTrue(serializer.is_valid()) serializer.save() out = StringIO() - call_command( - 'retrieve_org_or_project_list', - stdout=out - ) + call_command("retrieve_org_or_project_list", stdout=out) expected_project_data = { self.project.name: { user.username: { - 'first_name': user.first_name, - 'last_name': user.last_name, - 'is_org': False, - 'role': 'editor' + "first_name": user.first_name, + "last_name": user.last_name, + "is_org": False, + "role": "editor", }, self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'is_org': False, - 'role': 'owner' - } + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "is_org": False, + "role": "owner", + }, } } expected_org_data = { self.organization.name: { self.user.username: { - 'first_name': self.user.first_name, - 'last_name': self.user.last_name, - 'role': 'owner' + "first_name": self.user.first_name, + "last_name": self.user.last_name, + "role": "owner", }, self.organization.user.username: { - 'first_name': self.organization.user.first_name, - 'last_name': self.organization.user.last_name, - 'role': 'owner' - } + "first_name": self.organization.user.first_name, + "last_name": self.organization.user.last_name, + "role": "owner", + }, } } @@ -66,31 +61,20 @@ def test_retrieve_org_or_project_list(self): expected_data.update(expected_project_data) expected_data.update(expected_org_data) - self.assertEqual( - expected_data, - json.loads(out.getvalue()) - ) + self.assertEqual(expected_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - project_ids=f'{self.project.id}', - stdout=out - ) - self.assertEqual( - expected_project_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", project_ids=f"{self.project.id}", stdout=out ) + self.assertEqual(expected_project_data, json.loads(out.getvalue())) out = StringIO() call_command( - 'retrieve_org_or_project_list', - organization_ids=f'{self.organization.id}', - stdout=out - ) - self.assertEqual( - expected_org_data, - json.loads(out.getvalue()) + "retrieve_org_or_project_list", + organization_ids=f"{self.organization.id}", + stdout=out, ) + self.assertEqual(expected_org_data, json.loads(out.getvalue())) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 79522e821a..cb64b0f4b8 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -1,11 +1,13 @@ +import six + from xml.dom import NotFoundErr + from django.conf import settings from django.core.files import File from django.core.validators import ValidationError from django.contrib.auth.models import User from django.http import Http404 from django.utils.translation import ugettext as _ -from django.utils import six from rest_framework import exceptions from rest_framework import mixins @@ -38,13 +40,13 @@ def _extract_uuid(text): if isinstance(text, six.string_types): - form_id_parts = text.split('/') + form_id_parts = text.split("/") if form_id_parts.__len__() < 2: - raise ValidationError(_(u"Invalid formId %s." % text)) + raise ValidationError(_("Invalid formId %s." % text)) text = form_id_parts[1] - text = text[text.find("@key="):-1].replace("@key=", "") + text = text[text.find("@key=") : -1].replace("@key=", "") if text.startswith("uuid:"): text = text.replace("uuid:", "") @@ -54,7 +56,7 @@ def _extract_uuid(text): def _extract_id_string(formId): if isinstance(formId, six.string_types): - return formId[0:formId.find('[')] + return formId[0 : formId.find("[")] return formId @@ -66,45 +68,49 @@ def _parse_int(num): pass -class BriefcaseViewset(mixins.CreateModelMixin, - mixins.RetrieveModelMixin, mixins.ListModelMixin, - viewsets.GenericViewSet): +class BriefcaseViewset( + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): """ Implements the [Briefcase Aggregate API](\ https://code.google.com/p/opendatakit/wiki/BriefcaseAggregateAPI). """ + authentication_classes = (DigestAuthentication,) filter_backends = (filters.AnonDjangoObjectPermissionFilter,) queryset = XForm.objects.all() - permission_classes = (permissions.IsAuthenticated, - ViewDjangoObjectPermissions) + permission_classes = (permissions.IsAuthenticated, ViewDjangoObjectPermissions) renderer_classes = (TemplateXMLRenderer, BrowsableAPIRenderer) serializer_class = XFormListSerializer - template_name = 'openrosa_response.xml' + template_name = "openrosa_response.xml" def get_object(self, queryset=None): - formId = self.request.GET.get('formId', '') + formId = self.request.GET.get("formId", "") id_string = _extract_id_string(formId) uuid = _extract_uuid(formId) - username = self.kwargs.get('username') + username = self.kwargs.get("username") - obj = get_object_or_404(Instance, - xform__user__username__iexact=username, - xform__id_string__iexact=id_string, - uuid=uuid) + obj = get_object_or_404( + Instance, + xform__user__username__iexact=username, + xform__id_string__iexact=id_string, + uuid=uuid, + ) self.check_object_permissions(self.request, obj.xform) return obj def filter_queryset(self, queryset): - username = self.kwargs.get('username') + username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication self.permission_denied(self.request) if username is not None and self.request.user.is_anonymous: - profile = get_object_or_404( - UserProfile, user__username__iexact=username) + profile = get_object_or_404(UserProfile, user__username__iexact=username) if profile.require_auth: # raises a permission denied exception, forces authentication @@ -114,27 +120,24 @@ def filter_queryset(self, queryset): else: queryset = super(BriefcaseViewset, self).filter_queryset(queryset) - formId = self.request.GET.get('formId', '') + formId = self.request.GET.get("formId", "") - if formId.find('[') != -1: + if formId.find("[") != -1: formId = _extract_id_string(formId) - xform_kwargs = { - 'queryset': queryset, - 'id_string__iexact': formId - } + xform_kwargs = {"queryset": queryset, "id_string__iexact": formId} if username: - xform_kwargs['user__username__iexact'] = username + xform_kwargs["user__username__iexact"] = username xform = get_form(xform_kwargs) self.check_object_permissions(self.request, xform) instances = Instance.objects.filter( - xform=xform, deleted_at__isnull=True).values( - 'pk', 'uuid') + xform=xform, deleted_at__isnull=True + ).values("pk", "uuid") if xform.encrypted: instances = instances.filter(media_all_received=True) - instances = instances.order_by('pk') - num_entries = self.request.GET.get('numEntries') - cursor = self.request.GET.get('cursor') + instances = instances.order_by("pk") + num_entries = self.request.GET.get("numEntries") + cursor = self.request.GET.get("cursor") cursor = _parse_int(cursor) if cursor: @@ -152,7 +155,7 @@ def filter_queryset(self, queryset): if instance_count > 0: last_instance = instances[instance_count - 1] - self.resumptionCursor = last_instance.get('pk') + self.resumptionCursor = last_instance.get("pk") elif instance_count == 0 and cursor: self.resumptionCursor = cursor else: @@ -161,23 +164,28 @@ def filter_queryset(self, queryset): return instances def create(self, request, *args, **kwargs): - if request.method.upper() == 'HEAD': - return Response(status=status.HTTP_204_NO_CONTENT, - headers=get_openrosa_headers(request), - template_name=self.template_name) - - xform_def = request.FILES.get('form_def_file', None) + if request.method.upper() == "HEAD": + return Response( + status=status.HTTP_204_NO_CONTENT, + headers=get_openrosa_headers(request), + template_name=self.template_name, + ) + + xform_def = request.FILES.get("form_def_file", None) response_status = status.HTTP_201_CREATED - username = kwargs.get('username') - form_user = (username and get_object_or_404(User, username=username)) \ - or request.user + username = kwargs.get("username") + form_user = ( + username and get_object_or_404(User, username=username) + ) or request.user - if not request.user.has_perm('can_add_xform', form_user.profile): + if not request.user.has_perm("can_add_xform", form_user.profile): raise exceptions.PermissionDenied( - detail=_(u"User %(user)s has no permission to add xforms to " - "account %(account)s" % - {'user': request.user.username, - 'account': form_user.username})) + detail=_( + "User %(user)s has no permission to add xforms to " + "account %(account)s" + % {"user": request.user.username, "account": form_user.username} + ) + ) data = {} if isinstance(xform_def, File): @@ -185,30 +193,34 @@ def create(self, request, *args, **kwargs): dd = publish_form(do_form_upload.publish_xform) if isinstance(dd, XForm): - data['message'] = _( - u"%s successfully published." % dd.id_string) + data["message"] = _("%s successfully published." % dd.id_string) else: - data['message'] = dd['text'] + data["message"] = dd["text"] response_status = status.HTTP_400_BAD_REQUEST else: - data['message'] = _(u"Missing xml file.") + data["message"] = _("Missing xml file.") response_status = status.HTTP_400_BAD_REQUEST - return Response(data, status=response_status, - headers=get_openrosa_headers(request, - location=False), - template_name=self.template_name) + return Response( + data, + status=response_status, + headers=get_openrosa_headers(request, location=False), + template_name=self.template_name, + ) def list(self, request, *args, **kwargs): self.object_list = self.filter_queryset(self.get_queryset()) - data = {'instances': self.object_list, - 'resumptionCursor': self.resumptionCursor} + data = { + "instances": self.object_list, + "resumptionCursor": self.resumptionCursor, + } - return Response(data, - headers=get_openrosa_headers(request, - location=False), - template_name='submissionList.xml') + return Response( + data, + headers=get_openrosa_headers(request, location=False), + template_name="submissionList.xml", + ) def retrieve(self, request, *args, **kwargs): self.object = self.get_object() @@ -216,52 +228,54 @@ def retrieve(self, request, *args, **kwargs): xml_obj = clean_and_parse_xml(self.object.xml) submission_xml_root_node = xml_obj.documentElement submission_xml_root_node.setAttribute( - 'instanceID', u'uuid:%s' % self.object.uuid) + "instanceID", "uuid:%s" % self.object.uuid + ) submission_xml_root_node.setAttribute( - 'submissionDate', self.object.date_created.isoformat() + "submissionDate", self.object.date_created.isoformat() ) if getattr(settings, "SUPPORT_BRIEFCASE_SUBMISSION_DATE", True): # Remove namespace attribute if any try: - submission_xml_root_node.removeAttribute('xmlns') + submission_xml_root_node.removeAttribute("xmlns") except NotFoundErr: pass data = { - 'submission_data': submission_xml_root_node.toxml(), - 'media_files': Attachment.objects.filter(instance=self.object), - 'host': request.build_absolute_uri().replace( - request.get_full_path(), '') + "submission_data": submission_xml_root_node.toxml(), + "media_files": Attachment.objects.filter(instance=self.object), + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), } return Response( data, headers=get_openrosa_headers(request, location=False), - template_name='downloadSubmission.xml' + template_name="downloadSubmission.xml", ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def manifest(self, request, *args, **kwargs): self.object = self.get_object() - object_list = MetaData.objects.filter(data_type='media', - object_id=self.object.id) + object_list = MetaData.objects.filter( + data_type="media", object_id=self.object.id + ) context = self.get_serializer_context() - serializer = XFormManifestSerializer(object_list, many=True, - context=context) + serializer = XFormManifestSerializer(object_list, many=True, context=context) - return Response(serializer.data, - headers=get_openrosa_headers(request, location=False)) + return Response( + serializer.data, headers=get_openrosa_headers(request, location=False) + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def media(self, request, *args, **kwargs): self.object = self.get_object() - pk = kwargs.get('metadata') + pk = kwargs.get("metadata") if not pk: raise Http404() meta_obj = get_object_or_404( - MetaData, data_type='media', xform=self.object, pk=pk) + MetaData, data_type="media", xform=self.object, pk=pk + ) return get_media_file_response(meta_obj) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 918b7a6980..47b205a123 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -3,6 +3,8 @@ from builtins import str as text from typing import Union +import six + from django.conf import settings from django.core.exceptions import PermissionDenied from django.db.models import Q @@ -10,7 +12,6 @@ from django.db.utils import DataError, OperationalError from django.http import Http404 from django.http import StreamingHttpResponse -from django.utils import six from django.utils import timezone from distutils.util import strtobool from django.utils.translation import ugettext as _ @@ -28,9 +29,7 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models import OsmData, MergedXForm from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.instance import ( - Instance, - FormInactiveError) +from onadata.apps.logger.models.instance import Instance, FormInactiveError from onadata.apps.logger.models.xform import XForm from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED from onadata.apps.messaging.serializers import send_message @@ -43,18 +42,23 @@ from onadata.libs.exceptions import EnketoError from onadata.libs.exceptions import NoRecordsPermission from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import CountOverridablePageNumberPagination -from onadata.libs.permissions import CAN_DELETE_SUBMISSION, \ - filter_queryset_xform_meta_perms, filter_queryset_xform_meta_perms_sql +from onadata.libs.permissions import ( + CAN_DELETE_SUBMISSION, + filter_queryset_xform_meta_perms, + filter_queryset_xform_meta_perms_sql, +) from onadata.libs.renderers import renderers from onadata.libs.serializers.data_serializer import ( - DataInstanceSerializer, DataInstanceXMLSerializer, - InstanceHistorySerializer) + DataInstanceSerializer, + DataInstanceXMLSerializer, + InstanceHistorySerializer, +) from onadata.libs.serializers.data_serializer import DataSerializer from onadata.libs.serializers.data_serializer import JsonDataSerializer from onadata.libs.serializers.data_serializer import OSMSerializer @@ -63,20 +67,20 @@ from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.viewer_tools import get_form_url, get_enketo_urls -SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] -SUBMISSION_RETRIEVAL_THRESHOLD = getattr(settings, - "SUBMISSION_RETRIEVAL_THRESHOLD", - 10000) +SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] +SUBMISSION_RETRIEVAL_THRESHOLD = getattr( + settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 +) BaseViewset = get_baseviewset_class() def get_data_and_form(kwargs): - data_id = text(kwargs.get('dataid')) + data_id = text(kwargs.get("dataid")) if not data_id.isdigit(): - raise ParseError(_(u"Data ID should be an integer")) + raise ParseError(_("Data ID should be an integer")) - return (data_id, kwargs.get('format')) + return (data_id, kwargs.get("format")) def delete_instance(instance, user): @@ -93,11 +97,14 @@ def delete_instance(instance, user): raise ParseError(text(e)) -class DataViewSet(AnonymousUserPublicFormsMixin, - AuthenticateHeaderMixin, - ETagsMixin, CacheControlMixin, - BaseViewset, - ModelViewSet): +class DataViewSet( + AnonymousUserPublicFormsMixin, + AuthenticateHeaderMixin, + ETagsMixin, + CacheControlMixin, + BaseViewset, + ModelViewSet, +): """ This endpoint provides access to submitted data. """ @@ -116,16 +123,18 @@ class DataViewSet(AnonymousUserPublicFormsMixin, renderers.FLOIPRenderer, ] - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.XFormOwnerFilter, - filters.DataFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.XFormOwnerFilter, + filters.DataFilter, + ) serializer_class = DataSerializer permission_classes = (XFormPermissions,) - lookup_field = 'pk' - lookup_fields = ('pk', 'dataid') + lookup_field = "pk" + lookup_fields = ("pk", "dataid") extra_lookup_fields = None data_count = None - public_data_endpoint = 'public' + public_data_endpoint = "public" pagination_class = CountOverridablePageNumberPagination queryset = XForm.objects.filter(deleted_at__isnull=True) @@ -134,24 +143,22 @@ def get_serializer_class(self): pk_lookup, dataid_lookup = self.lookup_fields pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - fmt = self.kwargs.get('format', self.request.GET.get("format")) + fmt = self.kwargs.get("format", self.request.GET.get("format")) sort = self.request.GET.get("sort") fields = self.request.GET.get("fields") if fmt == Attachment.OSM: serializer_class = OSMSerializer - elif fmt == 'geojson': + elif fmt == "geojson": serializer_class = GeoJsonSerializer - elif fmt == 'xml': + elif fmt == "xml": serializer_class = DataInstanceXMLSerializer - elif pk is not None and dataid is None \ - and pk != self.public_data_endpoint: + elif pk is not None and dataid is None and pk != self.public_data_endpoint: if sort or fields: serializer_class = JsonDataSerializer else: serializer_class = DataInstanceSerializer else: - serializer_class = \ - super(DataViewSet, self).get_serializer_class() + serializer_class = super(DataViewSet, self).get_serializer_class() return serializer_class @@ -165,42 +172,42 @@ def get_object(self, queryset=None): try: int(dataid) except ValueError: - raise ParseError(_(u"Invalid dataid %(dataid)s" - % {'dataid': dataid})) + raise ParseError(_("Invalid dataid %(dataid)s" % {"dataid": dataid})) if not obj.is_merged_dataset: - obj = get_object_or_404(Instance, pk=dataid, xform__pk=pk, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform__pk=pk, deleted_at__isnull=True + ) else: xforms = obj.mergedxform.xforms.filter(deleted_at__isnull=True) - pks = [xform_id - for xform_id in xforms.values_list('pk', flat=True)] + pks = [xform_id for xform_id in xforms.values_list("pk", flat=True)] - obj = get_object_or_404(Instance, pk=dataid, xform_id__in=pks, - deleted_at__isnull=True) + obj = get_object_or_404( + Instance, pk=dataid, xform_id__in=pks, deleted_at__isnull=True + ) return obj def _get_public_forms_queryset(self): - return XForm.objects.filter(Q(shared=True) | Q(shared_data=True), - deleted_at__isnull=True) + return XForm.objects.filter( + Q(shared=True) | Q(shared_data=True), deleted_at__isnull=True + ) def _filtered_or_shared_qs(self, qs, pk): filter_kwargs = {self.lookup_field: pk} - qs = qs.filter(**filter_kwargs).only('id', 'shared') + qs = qs.filter(**filter_kwargs).only("id", "shared") if not qs: - filter_kwargs['shared_data'] = True - qs = XForm.objects.filter(**filter_kwargs).only('id', 'shared') + filter_kwargs["shared_data"] = True + qs = XForm.objects.filter(**filter_kwargs).only("id", "shared") if not qs: - raise Http404(_(u"No data matches with given query.")) + raise Http404(_("No data matches with given query.")) return qs def filter_queryset(self, queryset, view=None): - qs = super(DataViewSet, self).filter_queryset( - queryset.only('id', 'shared')) + qs = super(DataViewSet, self).filter_queryset(queryset.only("id", "shared")) pk = self.kwargs.get(self.lookup_field) if pk: @@ -210,73 +217,81 @@ def filter_queryset(self, queryset, view=None): if pk == self.public_data_endpoint: qs = self._get_public_forms_queryset() else: - raise ParseError(_(u"Invalid pk %(pk)s" % {'pk': pk})) + raise ParseError(_("Invalid pk %(pk)s" % {"pk": pk})) else: qs = self._filtered_or_shared_qs(qs, pk) else: - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") qs = qs.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') + not_tagged = not_tagged.split(",") qs = qs.exclude(tags__name__in=not_tagged) return qs - @action(methods=['GET', 'POST', 'DELETE'], detail=True, - extra_lookup_fields=['label', ]) + @action( + methods=["GET", "POST", "DELETE"], + detail=True, + extra_lookup_fields=[ + "label", + ], + ) def labels(self, request, *args, **kwargs): http_status = status.HTTP_400_BAD_REQUEST self.object = instance = self.get_object() - if request.method == 'POST': + if request.method == "POST": add_tags_to_instance(request, instance) http_status = status.HTTP_201_CREATED tags = instance.tags - label = kwargs.get('label') + label = kwargs.get("label") - if request.method == 'GET' and label: - data = [tag['name'] for tag in - tags.filter(name=label).values('name')] + if request.method == "GET" and label: + data = [tag["name"] for tag in tags.filter(name=label).values("name")] - elif request.method == 'DELETE' and label: + elif request.method == "DELETE" and label: count = tags.count() tags.remove(label) # Accepted, label does not exist hence nothing removed - http_status = status.HTTP_200_OK if count > tags.count() \ + http_status = ( + status.HTTP_200_OK + if count > tags.count() else status.HTTP_404_NOT_FOUND + ) data = list(tags.names()) else: data = list(tags.names()) - if request.method == 'GET': + if request.method == "GET": http_status = status.HTTP_200_OK self.etag_data = data return Response(data, status=http_status) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def enketo(self, request, *args, **kwargs): self.object = self.get_object() data = {} if isinstance(self.object, XForm): - raise ParseError(_(u"Data id not provided.")) - elif(isinstance(self.object, Instance)): + raise ParseError(_("Data id not provided.")) + elif isinstance(self.object, Instance): if request.user.has_perm("change_xform", self.object.xform): - return_url = request.query_params.get('return_url') + return_url = request.query_params.get("return_url") form_url = get_form_url( request, self.object.xform.user.username, - xform_pk=self.object.xform.id) + xform_pk=self.object.xform.id, + ) if not return_url: - raise ParseError(_(u"return_url not provided.")) + raise ParseError(_("return_url not provided.")) try: data = get_enketo_urls( @@ -284,52 +299,52 @@ def enketo(self, request, *args, **kwargs): self.object.xform.id_string, instance_id=self.object.uuid, instance_xml=self.object.xml, - return_url=return_url) + return_url=return_url, + ) if "edit_url" in data: data["url"] = data.pop("edit_url") except EnketoError as e: raise ParseError(text(e)) else: - raise PermissionDenied(_(u"You do not have edit permissions.")) + raise PermissionDenied(_("You do not have edit permissions.")) self.etag_data = data return Response(data=data) def destroy(self, request, *args, **kwargs): - instance_ids = request.data.get('instance_ids') - delete_all_submissions = strtobool( - request.data.get('delete_all', 'False')) + instance_ids = request.data.get("instance_ids") + delete_all_submissions = strtobool(request.data.get("delete_all", "False")) self.object = self.get_object() if isinstance(self.object, XForm): if not instance_ids and not delete_all_submissions: - raise ParseError(_(u"Data id(s) not provided.")) + raise ParseError(_("Data id(s) not provided.")) else: initial_count = self.object.submission_count() if delete_all_submissions: # Update timestamp only for active records - self.object.instances.filter( - deleted_at__isnull=True).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user) + self.object.instances.filter(deleted_at__isnull=True).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, + ) else: - instance_ids = [ - x for x in instance_ids.split(',') if x.isdigit()] + instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] if not instance_ids: - raise ParseError(_(u"Invalid data ids were provided.")) + raise ParseError(_("Invalid data ids were provided.")) self.object.instances.filter( id__in=instance_ids, xform=self.object, # do not update this timestamp when the record have # already been deleted. - deleted_at__isnull=True + deleted_at__isnull=True, ).update( deleted_at=timezone.now(), date_modified=timezone.now(), - deleted_by=request.user) + deleted_by=request.user, + ) # updates the num_of_submissions for the form. after_count = self.object.submission_count(force_update=True) @@ -337,37 +352,39 @@ def destroy(self, request, *args, **kwargs): # update the date modified field of the project self.object.project.date_modified = timezone.now() - self.object.project.save(update_fields=['date_modified']) + self.object.project.save(update_fields=["date_modified"]) # send message send_message( - instance_id=instance_ids, target_id=self.object.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) + instance_id=instance_ids, + target_id=self.object.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) return Response( data={ - "message": - "%d records were deleted" % - number_of_records_deleted + "message": "%d records were deleted" % number_of_records_deleted }, - status=status.HTTP_200_OK + status=status.HTTP_200_OK, ) elif isinstance(self.object, Instance): - if request.user.has_perm( - CAN_DELETE_SUBMISSION, self.object.xform): + if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): instance_id = self.object.pk delete_instance(self.object, request.user) # send message send_message( - instance_id=instance_id, target_id=self.object.xform.id, - target_type=XFORM, user=request.user, - message_verb=SUBMISSION_DELETED) + instance_id=instance_id, + target_id=self.object.xform.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) else: - raise PermissionDenied(_(u"You do not have delete " - u"permissions.")) + raise PermissionDenied(_("You do not have delete " "permissions.")) return Response(status=status.HTTP_204_NO_CONTENT) @@ -375,50 +392,56 @@ def retrieve(self, request, *args, **kwargs): data_id, _format = get_data_and_form(kwargs) self.object = instance = self.get_object() - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": return Response(instance.json) - elif _format == 'xml': + elif _format == "xml": return Response(instance.xml) - elif _format == 'geojson': - return super(DataViewSet, self)\ - .retrieve(request, *args, **kwargs) + elif _format == "geojson": + return super(DataViewSet, self).retrieve(request, *args, **kwargs) elif _format == Attachment.OSM: serializer = self.get_serializer(instance.osm_data.all()) return Response(serializer.data) else: raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) + _( + "'%(_format)s' format unknown or not implemented!" + % {"_format": _format} + ) + ) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def history(self, request, *args, **kwargs): data_id, _format = get_data_and_form(kwargs) instance = self.get_object() # retrieve all history objects and return them - if _format == 'json' or _format is None or _format == 'debug': + if _format == "json" or _format is None or _format == "debug": instance_history = instance.submission_history.all() - serializer = InstanceHistorySerializer( - instance_history, many=True) + serializer = InstanceHistorySerializer(instance_history, many=True) return Response(serializer.data) else: raise ParseError( - _(u"'%(_format)s' format unknown or not implemented!" % - {'_format': _format})) + _( + "'%(_format)s' format unknown or not implemented!" + % {"_format": _format} + ) + ) def _set_pagination_headers( - self, xform: XForm, current_page: Union[int, str], - current_page_size: Union[int, str] = - SUBMISSION_RETRIEVAL_THRESHOLD): + self, + xform: XForm, + current_page: Union[int, str], + current_page_size: Union[int, str] = SUBMISSION_RETRIEVAL_THRESHOLD, + ): """ Sets the self.headers value for the viewset """ import math url = self.request.build_absolute_uri() - query = self.request.query_params.get('query') - base_url = url.split('?')[0] + query = self.request.query_params.get("query") + base_url = url.split("?")[0] if query: num_of_records = self.object_list.count() else: @@ -443,36 +466,33 @@ def _set_pagination_headers( if (current_page * current_page_size) < num_of_records: next_page_url = ( - f"{base_url}?page={current_page + 1}&" - f"page_size={current_page_size}") + f"{base_url}?page={current_page + 1}&" f"page_size={current_page_size}" + ) if current_page > 1: prev_page_url = ( - f"{base_url}?page={current_page - 1}" - f"&page_size={current_page_size}") + f"{base_url}?page={current_page - 1}" f"&page_size={current_page_size}" + ) last_page = math.ceil(num_of_records / current_page_size) if last_page != current_page and last_page != current_page + 1: - last_page_url = ( - f"{base_url}?page={last_page}&page_size={current_page_size}" - ) + last_page_url = f"{base_url}?page={last_page}&page_size={current_page_size}" if current_page != 1: - first_page_url = ( - f"{base_url}?page=1&page_size={current_page_size}" - ) + first_page_url = f"{base_url}?page=1&page_size={current_page_size}" - if not hasattr(self, 'headers'): + if not hasattr(self, "headers"): self.headers = {} for rel, link in ( - ('prev', prev_page_url), - ('next', next_page_url), - ('last', last_page_url), - ('first', first_page_url)): + ("prev", prev_page_url), + ("next", next_page_url), + ("last", last_page_url), + ("first", first_page_url), + ): if link: links.append(f'<{link}>; rel="{rel}"') - self.headers.update({'Link': ', '.join(links)}) + self.headers.update({"Link": ", ".join(links)}) def list(self, request, *args, **kwargs): fields = request.GET.get("fields") @@ -480,7 +500,7 @@ def list(self, request, *args, **kwargs): sort = request.GET.get("sort") start = parse_int(request.GET.get("start")) limit = parse_int(request.GET.get("limit")) - export_type = kwargs.get('format', request.GET.get("format")) + export_type = kwargs.get("format", request.GET.get("format")) lookup_field = self.lookup_field lookup = self.kwargs.get(lookup_field) is_public_request = lookup == self.public_data_endpoint @@ -494,117 +514,116 @@ def list(self, request, *args, **kwargs): if is_public_request: self.object_list = self._get_public_forms_queryset() elif lookup: - qs = self.filter_queryset( - self.get_queryset() - ).values_list('pk', 'is_merged_dataset') + qs = self.filter_queryset(self.get_queryset()).values_list( + "pk", "is_merged_dataset" + ) xform_id, is_merged_dataset = qs[0] if qs else (lookup, False) pks = [xform_id] if is_merged_dataset: merged_form = MergedXForm.objects.get(pk=xform_id) - qs = merged_form.xforms.filter( - deleted_at__isnull=True).values_list( - 'id', 'num_of_submissions') + qs = merged_form.xforms.filter(deleted_at__isnull=True).values_list( + "id", "num_of_submissions" + ) try: - pks, num_of_submissions = [ - list(value) for value in zip(*qs)] + pks, num_of_submissions = [list(value) for value in zip(*qs)] num_of_submissions = sum(num_of_submissions) except ValueError: pks, num_of_submissions = [], 0 else: - num_of_submissions = XForm.objects.get( - id=xform_id).num_of_submissions + num_of_submissions = XForm.objects.get(id=xform_id).num_of_submissions self.object_list = Instance.objects.filter( - xform_id__in=pks, deleted_at=None).only('json') + xform_id__in=pks, deleted_at=None + ).only("json") # Enable ordering for XForms with Submissions that are less # than the SUBMISSION_RETRIEVAL_THRESHOLD if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD: - self.object_list = self.object_list.order_by('id') + self.object_list = self.object_list.order_by("id") xform = self.get_object() - self.object_list = \ - filter_queryset_xform_meta_perms(xform, request.user, - self.object_list) - tags = self.request.query_params.get('tags') - not_tagged = self.request.query_params.get('not_tagged') + self.object_list = filter_queryset_xform_meta_perms( + xform, request.user, self.object_list + ) + tags = self.request.query_params.get("tags") + not_tagged = self.request.query_params.get("not_tagged") self.object_list = filters.InstanceFilter( - self.request.query_params, - queryset=self.object_list, - request=request + self.request.query_params, queryset=self.object_list, request=request ).qs if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") self.object_list = self.object_list.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): - not_tagged = not_tagged.split(',') - self.object_list = \ - self.object_list.exclude(tags__name__in=not_tagged) + not_tagged = not_tagged.split(",") + self.object_list = self.object_list.exclude(tags__name__in=not_tagged) if ( - export_type is None or - export_type in ['json', 'jsonp', 'debug', 'xml']) \ - and hasattr(self, 'object_list'): - return self._get_data(query, fields, sort, start, limit, - is_public_request) + export_type is None or export_type in ["json", "jsonp", "debug", "xml"] + ) and hasattr(self, "object_list"): + return self._get_data(query, fields, sort, start, limit, is_public_request) xform = self.get_object() - kwargs = {'instance__xform': xform} + kwargs = {"instance__xform": xform} if export_type == Attachment.OSM: if request.GET: self.set_object_list( - query, fields, sort, start, limit, is_public_request) - kwargs = {'instance__in': self.object_list} - osm_list = OsmData.objects.filter(**kwargs).order_by('instance') + query, fields, sort, start, limit, is_public_request + ) + kwargs = {"instance__in": self.object_list} + osm_list = OsmData.objects.filter(**kwargs).order_by("instance") page = self.paginate_queryset(osm_list) serializer = self.get_serializer(page) return Response(serializer.data) - elif export_type is None or export_type in ['json']: + elif export_type is None or export_type in ["json"]: # perform default viewset retrieve, no data export return super(DataViewSet, self).list(request, *args, **kwargs) - elif export_type == 'geojson': + elif export_type == "geojson": serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) return custom_response_handler(request, xform, query, export_type) - def set_object_list( - self, query, fields, sort, start, limit, is_public_request): + def set_object_list(self, query, fields, sort, start, limit, is_public_request): try: enable_etag = True if not is_public_request: xform = self.get_object() self.data_count = xform.num_of_submissions - enable_etag = self.data_count <\ - SUBMISSION_RETRIEVAL_THRESHOLD + enable_etag = self.data_count < SUBMISSION_RETRIEVAL_THRESHOLD where, where_params = get_where_clause(query) if where: - self.object_list = self.object_list.extra(where=where, - params=where_params) + self.object_list = self.object_list.extra( + where=where, params=where_params + ) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( - self.get_object(), self.request.user, self.object_list) + self.get_object(), self.request.user, self.object_list + ) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: - query = \ - filter_queryset_xform_meta_perms_sql(self.get_object(), - self.request.user, - query) + query = filter_queryset_xform_meta_perms_sql( + self.get_object(), self.request.user, query + ) self.object_list = query_data( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields, - json_only=not self.kwargs.get('format') == 'xml') + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, + json_only=not self.kwargs.get("format") == "xml", + ) except NoRecordsPermission: self.object_list = [] @@ -615,11 +634,14 @@ def set_object_list( self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( - xform, query=query, sort=sort, start_index=start, - limit=limit, fields=fields + xform, + query=query, + sort=sort, + start_index=start, + limit=limit, + fields=fields, ) - self.etag_hash = get_etag_hash_from_query( - records, sql, params) + self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: @@ -628,19 +650,18 @@ def set_object_list( def paginate_queryset(self, queryset): if self.paginator is None: return None - return self.paginator.paginate_queryset(queryset, - self.request, - view=self, - count=self.data_count) + return self.paginator.paginate_queryset( + queryset, self.request, view=self, count=self.data_count + ) def _get_data(self, query, fields, sort, start, limit, is_public_request): - self.set_object_list( - query, fields, sort, start, limit, is_public_request) + self.set_object_list(query, fields, sort, start, limit, is_public_request) - retrieval_threshold = getattr( - settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) - pagination_keys = [self.paginator.page_query_param, - self.paginator.page_size_query_param] + retrieval_threshold = getattr(settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000) + pagination_keys = [ + self.paginator.page_query_param, + self.paginator.page_size_query_param, + ] query_param_keys = self.request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) @@ -653,13 +674,11 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): if should_paginate: self.paginator.page_size = retrieval_threshold - if not isinstance(self.object_list, types.GeneratorType) and \ - should_paginate: - current_page = query_param_keys.get( - self.paginator.page_query_param, 1) + if not isinstance(self.object_list, types.GeneratorType) and should_paginate: + current_page = query_param_keys.get(self.paginator.page_query_param, 1) current_page_size = query_param_keys.get( - self.paginator.page_size_query_param, - retrieval_threshold) + self.paginator.page_size_query_param, retrieval_threshold + ) self._set_pagination_headers( self.get_object(), @@ -672,7 +691,7 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): except OperationalError: self.object_list = self.paginate_queryset(self.object_list) - STREAM_DATA = getattr(settings, 'STREAM_DATA', False) + STREAM_DATA = getattr(settings, "STREAM_DATA", False) if STREAM_DATA: response = self._get_streaming_response() else: @@ -685,24 +704,25 @@ def _get_streaming_response(self): """ Get a StreamingHttpResponse response object """ + def get_json_string(item): - return json.dumps( - item.json if isinstance(item, Instance) else item) + return json.dumps(item.json if isinstance(item, Instance) else item) - if self.kwargs.get('format') == 'xml': + if self.kwargs.get("format") == "xml": response = StreamingHttpResponse( renderers.InstanceXMLRenderer().stream_data( - self.object_list, self.get_serializer), - content_type="application/xml" + self.object_list, self.get_serializer + ), + content_type="application/xml", ) else: response = StreamingHttpResponse( json_stream(self.object_list, get_json_string), - content_type="application/json" + content_type="application/json", ) # calculate etag value and add it to response headers - if hasattr(self, 'etag_hash'): + if hasattr(self, "etag_hash"): self.set_etag_header(None, self.etag_hash) # set headers on streaming response diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 3efe759183..02d3eb36d1 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -3,6 +3,8 @@ import random from datetime import datetime +import six + from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError @@ -17,7 +19,7 @@ HttpResponseRedirect, StreamingHttpResponse, ) -from django.utils import six, timezone +from django.utils import timezone from django.utils.http import urlencode from django.utils.translation import ugettext as _ from django.views.decorators.cache import never_cache diff --git a/onadata/apps/logger/tests/test_transfer_project_command.py b/onadata/apps/logger/tests/test_transfer_project_command.py index f771d58585..06df9a4567 100644 --- a/onadata/apps/logger/tests/test_transfer_project_command.py +++ b/onadata/apps/logger/tests/test_transfer_project_command.py @@ -4,7 +4,7 @@ from django.contrib.auth import get_user_model from django.core.management import call_command -from django.utils.six import StringIO +from six import StringIO from onadata.apps.logger.models import Project, XForm from onadata.apps.main.tests.test_base import TestBase @@ -12,87 +12,94 @@ class TestMoveProjectToAnewOwner(TestBase): # pylint: disable=C0111 def test_successful_project_transfer(self): # pylint: disable=C0103 - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, + stdout=mock_stdout, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 0, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 5, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(0, Project.objects.filter(organization=user1).count()) + self.assertEqual(5, Project.objects.filter(organization=user2).count()) def test_single_project_transfer(self): - """"Test for a successful project transfer.""" + """ "Test for a successful project transfer.""" user_model = get_user_model() user_1_data = { - 'username': 'user1', - 'email': 'user1@test.com', - 'password': 'test_pass' + "username": "user1", + "email": "user1@test.com", + "password": "test_pass", } user_2_data = { - 'username': 'user2', - 'email': 'user2@test.com', - 'password': 'test_pass' + "username": "user2", + "email": "user2@test.com", + "password": "test_pass", } user1 = user_model.objects.create_user(**user_1_data) user2 = user_model.objects.create_user(**user_2_data) Project.objects.create( - name='Test_project_1', organization=user1, created_by=user1) + name="Test_project_1", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_2', organization=user1, created_by=user1) + name="Test_project_2", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_3', organization=user1, created_by=user1) + name="Test_project_3", organization=user1, created_by=user1 + ) Project.objects.create( - name='Test_project_4', organization=user1, created_by=user1) + name="Test_project_4", organization=user1, created_by=user1 + ) test_project = Project.objects.create( - name='Test_project_5', organization=user1, created_by=user1) + name="Test_project_5", organization=user1, created_by=user1 + ) mock_stdout = StringIO() sys.stdout = mock_stdout self.assertIsNotNone(test_project.id) call_command( - 'transferproject', current_owner='user1', new_owner='user2', - project_id=test_project.id + "transferproject", + current_owner="user1", + new_owner="user2", + project_id=test_project.id, ) - expected_output = 'Projects transferred successfully' + expected_output = "Projects transferred successfully" self.assertIn(expected_output, mock_stdout.getvalue()) - self.assertEqual( - 4, Project.objects.filter(organization=user1).count() - ) - self.assertEqual( - 1, Project.objects.filter(organization=user2).count() - ) + self.assertEqual(4, Project.objects.filter(organization=user1).count()) + self.assertEqual(1, Project.objects.filter(organization=user2).count()) test_project_refetched = Project.objects.get(id=test_project.id) self.assertEqual(user2, test_project_refetched.organization) @@ -100,12 +107,12 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 """Test the transfer of ownership of the XForms.""" xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) @@ -113,22 +120,22 @@ def test_xforms_are_transferred_as_well(self): # pylint: disable=C0103 user_model = get_user_model() user_data = { - 'username': 'user', - 'email': 'user@test.com', - 'password': 'test_pass' + "username": "user", + "email": "user@test.com", + "password": "test_pass", } new_owner = user_model.objects.create_user(**user_data) mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='bob', new_owner='user', - all_projects=True, stdout=mock_stdout + "transferproject", + current_owner="bob", + new_owner="user", + all_projects=True, + stdout=mock_stdout, ) - self.assertIn( - 'Projects transferred successfully\n', - mock_stdout.getvalue() - ) - bob = user_model.objects.get(username='bob') + self.assertIn("Projects transferred successfully\n", mock_stdout.getvalue()) + bob = user_model.objects.get(username="bob") bobs_forms = XForm.objects.filter(user=bob) new_owner_forms = XForm.objects.filter(user=new_owner) self.assertEqual(0, bobs_forms.count()) @@ -142,14 +149,16 @@ class TestUserValidation(TestBase): it's stdout is interfering with the other functions causing them to fail. stdout.flush() does not help. """ - def test_user_given_does_not_exist(self): # pylint: disable=C0103 + + def test_user_given_does_not_exist(self): # pylint: disable=C0103 """Test that users are validated before initiating project transfer""" mock_stdout = StringIO() sys.stdout = mock_stdout call_command( - 'transferproject', current_owner='user1', new_owner='user2', - all_projects=True + "transferproject", + current_owner="user1", + new_owner="user2", + all_projects=True, ) - expected_output = 'User user1 does not existUser user2 does '\ - 'not exist\n' + expected_output = "User user1 does not existUser user2 does " "not exist\n" self.assertEqual(mock_stdout.getvalue(), expected_output) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index cef962db4c..11b3a9e849 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -9,6 +9,7 @@ from datetime import datetime import pytz +import six from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -16,16 +17,18 @@ from django.contrib.sites.models import Site from django.core.files import File from django.core.files.storage import get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseRedirect, +) from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader from django.urls import reverse -from django.utils import six from django.utils.translation import ugettext as _ from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import (require_GET, require_http_methods, - require_POST) +from django.views.decorators.http import require_GET, require_http_methods, require_POST from django_digest import HttpDigestAuthenticator from onadata.apps.logger.import_tools import import_instances_from_zip @@ -36,21 +39,28 @@ from onadata.libs.exceptions import EnketoError from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.cache_tools import (cache, - USER_PROFILE_PREFIX) +from onadata.libs.utils.cache_tools import cache, USER_PROFILE_PREFIX from onadata.libs.utils.logger_tools import ( - BaseOpenRosaResponse, OpenRosaResponse, OpenRosaResponseBadRequest, - PublishXForm, inject_instanceid, publish_form, remove_xform, - response_with_mimetype_and_name, safe_create_instance) + BaseOpenRosaResponse, + OpenRosaResponse, + OpenRosaResponseBadRequest, + PublishXForm, + inject_instanceid, + publish_form, + remove_xform, + response_with_mimetype_and_name, + safe_create_instance, +) from onadata.libs.utils.user_auth import ( - HttpResponseNotAuthorized, add_cors_headers, has_edit_permission, - has_permission, helper_auth_helper) -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form, get_form_url) + HttpResponseNotAuthorized, + add_cors_headers, + has_edit_permission, + has_permission, + helper_auth_helper, +) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form, get_form_url -IO_ERROR_STRINGS = [ - 'request data read error', 'error during read(65536) on wsgi.input' -] +IO_ERROR_STRINGS = ["request data read error", "error during read(65536) on wsgi.input"] def _bad_request(e): @@ -60,7 +70,7 @@ def _bad_request(e): def _extract_uuid(text): - text = text[text.find("@key="):-1].replace("@key=", "") + text = text[text.find("@key=") : -1].replace("@key=", "") if text.startswith("uuid:"): text = text.replace("uuid:", "") return text @@ -75,24 +85,23 @@ def _parse_int(num): def _html_submission_response(request, instance): data = {} - data['username'] = instance.xform.user.username - data['id_string'] = instance.xform.id_string - data['domain'] = Site.objects.get(id=settings.SITE_ID).domain + data["username"] = instance.xform.user.username + data["id_string"] = instance.xform.id_string + data["domain"] = Site.objects.get(id=settings.SITE_ID).domain return render(request, "submission.html", data) def _submission_response(instance): data = {} - data['message'] = _("Successful submission.") - data['formid'] = instance.xform.id_string - data['encrypted'] = instance.xform.encrypted - data['instanceID'] = u'uuid:%s' % instance.uuid - data['submissionDate'] = instance.date_created.isoformat() - data['markedAsCompleteDate'] = instance.date_modified.isoformat() + data["message"] = _("Successful submission.") + data["formid"] = instance.xform.id_string + data["encrypted"] = instance.xform.encrypted + data["instanceID"] = "uuid:%s" % instance.uuid + data["submissionDate"] = instance.date_created.isoformat() + data["markedAsCompleteDate"] = instance.date_modified.isoformat() - return BaseOpenRosaResponse( - loader.get_template('submission.xml').render(data)) + return BaseOpenRosaResponse(loader.get_template("submission.xml").render(data)) @require_POST @@ -112,25 +121,30 @@ def bulksubmission(request, username): temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest( - _(u"There was a problem receiving your " - u"ODK submission. [Error: IO Error " - u"reading data]")) + _( + "There was a problem receiving your " + "ODK submission. [Error: IO Error " + "reading data]" + ) + ) if len(temp_postfile) != 1: return HttpResponseBadRequest( - _(u"There was a problem receiving your" - u" ODK submission. [Error: multiple " - u"submission files (?)]")) + _( + "There was a problem receiving your" + " ODK submission. [Error: multiple " + "submission files (?)]" + ) + ) postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) - with open(our_tfpath, 'wb') as f: + with open(our_tfpath, "wb") as f: f.write(postfile.read()) - with open(our_tfpath, 'rb') as f: - total_count, success_count, errors = import_instances_from_zip( - f, posting_user) + with open(our_tfpath, "rb") as f: + total_count, success_count, errors = import_instances_from_zip(f, posting_user) # chose the try approach as suggested by the link below # http://stackoverflow.com/questions/82831 try: @@ -138,22 +152,31 @@ def bulksubmission(request, username): except IOError: pass json_msg = { - 'message': _(u"Submission complete. Out of %(total)d " - u"survey instances, %(success)d were imported, " - u"(%(rejected)d were rejected as duplicates, " - u"missing forms, etc.)") % { - 'total': total_count, - 'success': success_count, - 'rejected': total_count - success_count - }, - 'errors': u"%d %s" % (len(errors), errors) + "message": _( + "Submission complete. Out of %(total)d " + "survey instances, %(success)d were imported, " + "(%(rejected)d were rejected as duplicates, " + "missing forms, etc.)" + ) + % { + "total": total_count, + "success": success_count, + "rejected": total_count - success_count, + }, + "errors": "%d %s" % (len(errors), errors), } audit = {"bulk_submission_log": json_msg} - audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user, - _("Made bulk submissions."), audit, request) + audit_log( + Actions.USER_BULK_SUBMISSION, + request.user, + posting_user, + _("Made bulk submissions."), + audit, + request, + ) response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response @@ -164,9 +187,9 @@ def bulksubmission_form(request, username=None): """ username = username if username is None else username if request.user.username == username: - return render(request, 'bulk_submission_form.html') + return render(request, "bulk_submission_form.html") - return HttpResponseRedirect('/%s' % request.user.username) + return HttpResponseRedirect("/%s" % request.user.username) @require_GET @@ -175,11 +198,11 @@ def formList(request, username): # pylint: disable=C0103 formList view, /formList OpenRosa Form Discovery API 1.0. """ formlist_user = get_object_or_404(User, username__iexact=username) - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -195,33 +218,37 @@ def formList(request, username): # pylint: disable=C0103 # for users who are non-owner if request.user.username == profile.user.username: xforms = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True, - user__username__iexact=username) + downloadable=True, deleted_at__isnull=True, user__username__iexact=username + ) else: xforms = XForm.objects.filter( downloadable=True, deleted_at__isnull=True, user__username__iexact=username, - require_auth=False) + require_auth=False, + ) audit = {} - audit_log(Actions.USER_FORMLIST_REQUESTED, request.user, formlist_user, - _("Requested forms list."), audit, request) + audit_log( + Actions.USER_FORMLIST_REQUESTED, + request.user, + formlist_user, + _("Requested forms list."), + audit, + request, + ) data = { - 'host': request.build_absolute_uri().replace(request.get_full_path(), - ''), - 'xforms': xforms + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "xforms": xforms, } response = render( - request, - "xformsList.xml", - data, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + request, "xformsList.xml", data, content_type="text/xml; charset=utf-8" + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response @@ -231,18 +258,15 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 """ XFormManifest view, part of OpenRosa Form Discovery API 1.0. """ - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) formlist_user = xform.user - profile = cache.get( - f'{USER_PROFILE_PREFIX}{formlist_user.username}') + profile = cache.get(f"{USER_PROFILE_PREFIX}{formlist_user.username}") if not profile: profile, __ = UserProfile.objects.get_or_create( - user__username=formlist_user.username) + user__username=formlist_user.username + ) if profile.require_auth: authenticator = HttpDigestAuthenticator() @@ -251,16 +275,17 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 response = render( request, - "xformsManifest.xml", { - 'host': - request.build_absolute_uri().replace(request.get_full_path(), ''), - 'media_files': - MetaData.media_upload(xform, download=True) + "xformsManifest.xml", + { + "host": request.build_absolute_uri().replace(request.get_full_path(), ""), + "media_files": MetaData.media_upload(xform, download=True), }, - content_type="text/xml; charset=utf-8") - response['X-OpenRosa-Version'] = '1.0' - response['Date'] = datetime.now(pytz.timezone(settings.TIME_ZONE))\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') + content_type="text/xml; charset=utf-8", + ) + response["X-OpenRosa-Version"] = "1.0" + response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( + "%a, %d %b %Y %H:%M:%S %Z" + ) return response @@ -280,14 +305,16 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 if not authenticator.authenticate(request): return authenticator.build_challenge_response() - if request.method == 'HEAD': + if request.method == "HEAD": response = OpenRosaResponse(status=204) if username: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/submission' % username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/%s/submission" % username + ) else: - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/submission') + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/submission" + ) return response xml_file_list = [] @@ -299,27 +326,33 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 xml_file_list = request.FILES.pop("xml_submission_file", []) if len(xml_file_list) != 1: return OpenRosaResponseBadRequest( - _(u"There should be a single XML submission file.")) + _("There should be a single XML submission file.") + ) # save this XML file and media files as attachments media_files = request.FILES.values() # get uuid from post request - uuid = request.POST.get('uuid') + uuid = request.POST.get("uuid") - error, instance = safe_create_instance(username, xml_file_list[0], - media_files, uuid, request) + error, instance = safe_create_instance( + username, xml_file_list[0], media_files, uuid, request + ) if error: return error elif instance is None: - return OpenRosaResponseBadRequest( - _(u"Unable to create submission.")) + return OpenRosaResponseBadRequest(_("Unable to create submission.")) audit = {"xform": instance.xform.id_string} - audit_log(Actions.SUBMISSION_CREATED, request.user, - instance.xform.user, - _("Created submission on form %(id_string)s.") % - {"id_string": instance.xform.id_string}, audit, request) + audit_log( + Actions.SUBMISSION_CREATED, + request.user, + instance.xform.user, + _("Created submission on form %(id_string)s.") + % {"id_string": instance.xform.id_string}, + audit, + request, + ) # response as html if posting with a UUID if not username and uuid: @@ -331,12 +364,11 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 # 1) the status code needs to be 201 (created) # 2) The location header needs to be set to the host it posted to response.status_code = 201 - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) return response except IOError as e: if _bad_request(e): - return OpenRosaResponseBadRequest( - _(u"File transfer interruption.")) + return OpenRosaResponseBadRequest(_("File transfer interruption.")) else: raise finally: @@ -351,7 +383,7 @@ def download_xform(request, username, id_string): Download XForm XML view. """ user = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': user, 'id_string__iexact': id_string}) + xform = get_form({"user": user, "id_string__iexact": id_string}) profile, __ = UserProfile.objects.get_or_create(user=user) if profile.require_auth: @@ -359,11 +391,15 @@ def download_xform(request, username, id_string): if not authenticator.authenticate(request): return authenticator.build_challenge_response() audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XML_DOWNLOADED, request.user, xform.user, - _("Downloaded XML for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) - response = response_with_mimetype_and_name( - 'xml', id_string, show_date=False) + audit_log( + Actions.FORM_XML_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XML for form '%(id_string)s'.") % {"id_string": xform.id_string}, + audit, + request, + ) + response = response_with_mimetype_and_name("xml", id_string, show_date=False) response.content = xform.xml return response @@ -372,44 +408,53 @@ def download_xlsform(request, username, id_string): """ Download XLSForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = User.objects.get(username__iexact=username) helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") file_path = xform.xls.name default_storage = get_storage_class()() - if file_path != '' and default_storage.exists(file_path): + if file_path != "" and default_storage.exists(file_path): audit = {"xform": xform.id_string} - audit_log(Actions.FORM_XLS_DOWNLOADED, request.user, xform.user, - _("Downloaded XLS file for form '%(id_string)s'.") % - {"id_string": xform.id_string}, audit, request) + audit_log( + Actions.FORM_XLS_DOWNLOADED, + request.user, + xform.user, + _("Downloaded XLS file for form '%(id_string)s'.") + % {"id_string": xform.id_string}, + audit, + request, + ) split_path = file_path.split(os.extsep) - extension = 'xls' + extension = "xls" if len(split_path) > 1: extension = split_path[len(split_path) - 1] response = response_with_mimetype_and_name( - 'vnd.ms-excel', + "vnd.ms-excel", id_string, show_date=False, extension=extension, - file_path=file_path) + file_path=file_path, + ) return response else: - messages.add_message(request, messages.WARNING, - _(u'No XLS file for your form ' - u'%(id)s') % {'id': id_string}) + messages.add_message( + request, + messages.WARNING, + _("No XLS file for your form " "%(id)s") + % {"id": id_string}, + ) return HttpResponseRedirect("/%s" % username) @@ -419,10 +464,9 @@ def download_jsonform(request, username, id_string): XForm JSON view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if request.method == "OPTIONS": response = HttpResponse() @@ -430,13 +474,12 @@ def download_jsonform(request, username, id_string): return response helper_auth_helper(request) if not has_permission(xform, owner, request, xform.shared): - response = HttpResponseForbidden(_(u'Not shared.')) + response = HttpResponseForbidden(_("Not shared.")) add_cors_headers(response) return response - response = response_with_mimetype_and_name( - 'json', id_string, show_date=False) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') + response = response_with_mimetype_and_name("json", id_string, show_date=False) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") response.content = "%s(%s)" % (callback, xform.json) else: add_cors_headers(response) @@ -450,20 +493,26 @@ def delete_xform(request, username, id_string): """ Delete XForm view. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) # delete xform and submissions remove_xform(xform) audit = {} - audit_log(Actions.FORM_DELETED, request.user, xform.user, - _("Deleted form '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect('/') + audit_log( + Actions.FORM_DELETED, + request.user, + xform.user, + _("Deleted form '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + return HttpResponseRedirect("/") @is_owner @@ -471,21 +520,26 @@ def toggle_downloadable(request, username, id_string): """ Toggle XForm view, changes downloadable status of a form. """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) xform.downloadable = not xform.downloadable xform.save() audit = {} - audit_log(Actions.FORM_UPDATED, request.user, xform.user, - _("Made form '%(id_string)s' %(downloadable)s.") % { - 'id_string': - xform.id_string, - 'downloadable': - _("downloadable") - if xform.downloadable else _("un-downloadable") - }, audit, request) + audit_log( + Actions.FORM_UPDATED, + request.user, + xform.user, + _("Made form '%(id_string)s' %(downloadable)s.") + % { + "id_string": xform.id_string, + "downloadable": _("downloadable") + if xform.downloadable + else _("un-downloadable"), + }, + audit, + request, + ) return HttpResponseRedirect("/%s" % username) @@ -494,48 +548,47 @@ def enter_data(request, username, id_string): Redirects to Enketo webform view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: enketo_urls = get_enketo_urls(form_url, xform.id_string) - url = enketo_urls.get('url') + url = enketo_urls.get("url") if not url: return HttpResponseRedirect( reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + "form-show", kwargs={"username": username, "id_string": id_string} + ) + ) return HttpResponseRedirect(url) except EnketoError as e: data = {} owner = User.objects.get(username__iexact=username) - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - data['xform'] = xform - data['content_user'] = owner - data['form_view'] = True - data['message'] = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + data["xform"] = xform + data["content_user"] = owner + data["form_view"] = True + data["message"] = { + "type": "alert-error", + "text": "Enketo error, reason: %s" % e, } messages.add_message( request, messages.WARNING, _("Enketo error: enketo replied %s") % e, - fail_silently=True) + fail_silently=True, + ) return render(request, "profile.html", data) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def edit_data(request, username, id_string, data_id): @@ -544,30 +597,27 @@ def edit_data(request, username, id_string, data_id): """ context = RequestContext(request) owner = User.objects.get(username__iexact=username) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) instance = get_object_or_404(Instance, pk=data_id, xform=xform) if not has_edit_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) - if not hasattr(settings, 'ENKETO_URL'): + return HttpResponseForbidden(_("Not shared.")) + if not hasattr(settings, "ENKETO_URL"): return HttpResponseRedirect( - reverse( - 'form-show', - kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) - url = '%sdata/edit_url' % settings.ENKETO_URL + url = "%sdata/edit_url" % settings.ENKETO_URL # see commit 220f2dad0e for tmp file creation injected_xml = inject_instanceid(instance.xml, instance.uuid) return_url = request.build_absolute_uri( reverse( - 'submission-instance', - kwargs={'username': username, - 'id_string': id_string}) + "#/" + text(instance.id)) + "submission-instance", kwargs={"username": username, "id_string": id_string} + ) + + "#/" + + text(instance.id) + ) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) try: @@ -576,26 +626,27 @@ def edit_data(request, username, id_string, data_id): xform.id_string, instance_xml=injected_xml, instance_id=instance.uuid, - return_url=return_url) + return_url=return_url, + ) except EnketoError as e: context.message = { - 'type': 'alert-error', - 'text': u"Enketo error, reason: %s" % e + "type": "alert-error", + "text": "Enketo error, reason: %s" % e, } messages.add_message( request, messages.WARNING, _("Enketo error: enketo replied %s") % e, - fail_silently=True) + fail_silently=True, + ) else: if url: - url = url['edit_url'] + url = url["edit_url"] context.enketo = url return HttpResponseRedirect(url) return HttpResponseRedirect( - reverse( - 'form-show', kwargs={'username': username, - 'id_string': id_string})) + reverse("form-show", kwargs={"username": username, "id_string": id_string}) + ) def view_submission_list(request, username): @@ -609,18 +660,15 @@ def view_submission_list(request, username): authenticator = HttpDigestAuthenticator() if not authenticator.authenticate(request): return authenticator.build_challenge_response() - id_string = request.GET.get('formId', None) - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + id_string = request.GET.get("formId", None) + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') - num_entries = request.GET.get('numEntries', None) - cursor = request.GET.get('cursor', None) - instances = xform.instances.filter(deleted_at=None).order_by('pk') + return HttpResponseForbidden("Not shared.") + num_entries = request.GET.get("numEntries", None) + cursor = request.GET.get("cursor", None) + instances = xform.instances.filter(deleted_at=None).order_by("pk") cursor = _parse_int(cursor) if cursor: @@ -630,7 +678,7 @@ def view_submission_list(request, username): if num_entries: instances = instances[:num_entries] - data = {'instances': instances} + data = {"instances": instances} resumption_cursor = 0 if instances.count(): @@ -639,13 +687,11 @@ def view_submission_list(request, username): elif instances.count() == 0 and cursor: resumption_cursor = cursor - data['resumptionCursor'] = resumption_cursor + data["resumptionCursor"] = resumption_cursor return render( - request, - 'submissionList.xml', - data, - content_type="text/xml; charset=utf-8") + request, "submissionList.xml", data, content_type="text/xml; charset=utf-8" + ) def view_download_submission(request, username): @@ -660,12 +706,12 @@ def view_download_submission(request, username): if not authenticator.authenticate(request): return authenticator.build_challenge_response() data = {} - form_id = request.GET.get('formId', None) + form_id = request.GET.get("formId", None) if not isinstance(form_id, six.string_types): return HttpResponseBadRequest() - id_string = form_id[0:form_id.find('[')] - form_id_parts = form_id.split('/') + id_string = form_id[0 : form_id.find("[")] + form_id_parts = form_id.split("/") if form_id_parts.__len__() < 2: return HttpResponseBadRequest() @@ -675,25 +721,23 @@ def view_download_submission(request, username): xform__id_string__iexact=id_string, uuid=uuid, xform__user__username=username, - deleted_at__isnull=True) + deleted_at__isnull=True, + ) xform = instance.xform if not has_permission(xform, form_user, request, xform.shared_data): - return HttpResponseForbidden('Not shared.') + return HttpResponseForbidden("Not shared.") submission_xml_root_node = instance.get_root_node() - submission_xml_root_node.setAttribute('instanceID', - u'uuid:%s' % instance.uuid) - submission_xml_root_node.setAttribute('submissionDate', - instance.date_created.isoformat()) - data['submission_data'] = submission_xml_root_node.toxml() - data['media_files'] = Attachment.objects.filter(instance=instance) - data['host'] = request.build_absolute_uri().replace( - request.get_full_path(), '') + submission_xml_root_node.setAttribute("instanceID", "uuid:%s" % instance.uuid) + submission_xml_root_node.setAttribute( + "submissionDate", instance.date_created.isoformat() + ) + data["submission_data"] = submission_xml_root_node.toxml() + data["media_files"] = Attachment.objects.filter(instance=instance) + data["host"] = request.build_absolute_uri().replace(request.get_full_path(), "") return render( - request, - 'downloadSubmission.xml', - data, - content_type="text/xml; charset=utf-8") + request, "downloadSubmission.xml", data, content_type="text/xml; charset=utf-8" + ) @require_http_methods(["HEAD", "POST"]) @@ -711,23 +755,27 @@ def form_upload(request, username): return authenticator.build_challenge_response() if form_user != request.user: return HttpResponseForbidden( - _(u"Not allowed to upload form[s] to %(user)s account." % - {'user': form_user})) - if request.method == 'HEAD': + _( + "Not allowed to upload form[s] to %(user)s account." + % {"user": form_user} + ) + ) + if request.method == "HEAD": response = OpenRosaResponse(status=204) - response['Location'] = request.build_absolute_uri().replace( - request.get_full_path(), '/%s/formUpload' % form_user.username) + response["Location"] = request.build_absolute_uri().replace( + request.get_full_path(), "/%s/formUpload" % form_user.username + ) return response - xform_def = request.FILES.get('form_def_file', None) - content = u"" + xform_def = request.FILES.get("form_def_file", None) + content = "" if isinstance(xform_def, File): do_form_upload = PublishXForm(xform_def, form_user) xform = publish_form(do_form_upload.publish_xform) status = 201 if isinstance(xform, XForm): - content = _(u"%s successfully published." % xform.id_string) + content = _("%s successfully published." % xform.id_string) else: - content = xform['text'] + content = xform["text"] if isinstance(content, Exception): content = content status = 500 diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 9c14884361..8be56809c6 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,3 +1,5 @@ +import six + from uuid import UUID from django.contrib.auth.models import User @@ -6,7 +8,6 @@ from django.db.models import Q from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.utils import six from django_filters import rest_framework as django_filter_filters from rest_framework import filters from rest_framework_guardian.filters import ObjectPermissionsFilter @@ -16,18 +17,15 @@ from onadata.apps.viewer.models import Export from onadata.libs.utils.numeric import int_or_parse_error from onadata.libs.utils.common_tags import MEDIA_FILE_TYPES -from onadata.libs.permissions import \ - exclude_items_from_queryset_using_xform_meta_perms +from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms class AnonDjangoObjectPermissionFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): """ Anonymous user has no object permissions, return queryset as it is. """ - form_id = view.kwargs.get( - view.lookup_field, view.kwargs.get('xform_pk')) + form_id = view.kwargs.get(view.lookup_field, view.kwargs.get("xform_pk")) lookup_field = view.lookup_field queryset = queryset.filter(deleted_at=None) @@ -35,16 +33,15 @@ def filter_queryset(self, request, queryset, view): return queryset if form_id: - if lookup_field == 'pk': - int_or_parse_error(form_id, - u'Invalid form ID. It must be a positive' - ' integer') + if lookup_field == "pk": + int_or_parse_error( + form_id, "Invalid form ID. It must be a positive" " integer" + ) try: - if lookup_field == 'uuid': + if lookup_field == "uuid": form_id = UUID(form_id) - form = queryset.get( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + form = queryset.get(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) else: xform_kwargs = {lookup_field: form_id} form = queryset.get(**xform_kwargs) @@ -53,14 +50,14 @@ def filter_queryset(self, request, queryset, view): # Check if form is public and return it if form.shared: - if lookup_field == 'uuid': - return queryset.filter( - Q(uuid=form_id.hex) | Q(uuid=str(form_id))) + if lookup_field == "uuid": + return queryset.filter(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) else: return queryset.filter(Q(**xform_kwargs)) - return super(AnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + return super(AnonDjangoObjectPermissionFilter, self).filter_queryset( + request, queryset, view + ) # pylint: disable=too-few-public-methods @@ -73,20 +70,22 @@ class EnketoAnonDjangoObjectPermissionFilter(AnonDjangoObjectPermissionFilter): def filter_queryset(self, request, queryset, view): """Check report_xform permission when requesting for Enketo URL.""" - if view.action == 'enketo': - self.perm_format = '%(app_label)s.report_%(model_name)s' # noqa pylint: disable=W0201 - return super(EnketoAnonDjangoObjectPermissionFilter, self)\ - .filter_queryset(request, queryset, view) + if view.action == "enketo": + self.perm_format = ( + "%(app_label)s.report_%(model_name)s" # noqa pylint: disable=W0201 + ) + return super(EnketoAnonDjangoObjectPermissionFilter, self).filter_queryset( + request, queryset, view + ) class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): - perm_format = '%(app_label)s.report_%(model_name)s' + perm_format = "%(app_label)s.report_%(model_name)s" class XFormListXFormPKFilter(object): - def filter_queryset(self, request, queryset, view): - xform_pk = view.kwargs.get('xform_pk') + xform_pk = view.kwargs.get("xform_pk") if xform_pk: try: xform_pk = int(xform_pk) @@ -105,38 +104,36 @@ class FormIDFilter(django_filter_filters.FilterSet): class Meta: model = XForm - fields = ['formID'] + fields = ["formID"] class OrganizationPermissionFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): """Return a filtered queryset or all profiles if a getting a specific - profile.""" - if view.action == 'retrieve' and request.method == 'GET': + profile.""" + if view.action == "retrieve" and request.method == "GET": return queryset.model.objects.all() - filtered_queryset = super(OrganizationPermissionFilter, self)\ - .filter_queryset(request, queryset, view) - org_users = set([group.team.organization - for group in request.user.groups.all()] + [ - o.user for o in filtered_queryset]) + filtered_queryset = super(OrganizationPermissionFilter, self).filter_queryset( + request, queryset, view + ) + org_users = set( + [group.team.organization for group in request.user.groups.all()] + + [o.user for o in filtered_queryset] + ) - return queryset.model.objects.filter(user__in=org_users, - user__is_active=True) + return queryset.model.objects.filter(user__in=org_users, user__is_active=True) class XFormOwnerFilter(filters.BaseFilterBackend): - owner_prefix = 'user' + owner_prefix = "user" def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) @@ -144,7 +141,6 @@ def filter_queryset(self, request, queryset, view): class DataFilter(ObjectPermissionsFilter): - def filter_queryset(self, request, queryset, view): if request.user.is_anonymous: return queryset.filter(Q(shared_data=True)) @@ -155,57 +151,76 @@ class InstanceFilter(django_filter_filters.FilterSet): """ Instance FilterSet implemented using django-filter """ + submitted_by__id = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='id', + to_field_name="id", ) submitted_by__username = django_filter_filters.ModelChoiceFilter( - field_name='user', + field_name="user", queryset=User.objects.all(), - to_field_name='username', + to_field_name="username", ) media_all_received = django_filter_filters.BooleanFilter() class Meta: model = Instance - date_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte', 'year', - 'year__gt', 'year__lt', 'year__gte', 'year__lte', - 'month', 'month__gt', 'month__lt', 'month__gte', - 'month__lte', 'day', 'day__gt', 'day__lt', - 'day__gte', 'day__lte'] - generic_field_lookups = ['exact', 'gt', 'lt', 'gte', 'lte'] - fields = {'date_created': date_field_lookups, - 'date_modified': date_field_lookups, - 'last_edited': date_field_lookups, - 'media_all_received': ['exact'], - 'status': ['exact'], - 'survey_type__slug': ['exact'], - 'user__id': ['exact'], - 'user__username': ['exact'], - 'uuid': ['exact'], - 'version': generic_field_lookups} + date_field_lookups = [ + "exact", + "gt", + "lt", + "gte", + "lte", + "year", + "year__gt", + "year__lt", + "year__gte", + "year__lte", + "month", + "month__gt", + "month__lt", + "month__gte", + "month__lte", + "day", + "day__gt", + "day__lt", + "day__gte", + "day__lte", + ] + generic_field_lookups = ["exact", "gt", "lt", "gte", "lte"] + fields = { + "date_created": date_field_lookups, + "date_modified": date_field_lookups, + "last_edited": date_field_lookups, + "media_all_received": ["exact"], + "status": ["exact"], + "survey_type__slug": ["exact"], + "user__id": ["exact"], + "user__username": ["exact"], + "uuid": ["exact"], + "version": generic_field_lookups, + } class ProjectOwnerFilter(filters.BaseFilterBackend): - owner_prefix = 'organization' + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): - owner = request.query_params.get('owner') + owner = request.query_params.get("owner") if owner: - kwargs = { - self.owner_prefix + '__username__iexact': owner - } + kwargs = {self.owner_prefix + "__username__iexact": owner} return queryset.filter(**kwargs) | Project.objects.filter( - shared=True, deleted_at__isnull=True, **kwargs) + shared=True, deleted_at__isnull=True, **kwargs + ) return queryset class AnonUserProjectFilter(ObjectPermissionsFilter): - owner_prefix = 'organization' + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): """ @@ -218,9 +233,10 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(Q(shared=True)) if project_id: - int_or_parse_error(project_id, - u"Invalid value for project_id. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for project_id. It must be a" " positive integer.", + ) # check if project is public and return it try: @@ -231,38 +247,36 @@ def filter_queryset(self, request, queryset, view): if project.shared: return queryset.filter(Q(id=project_id)) - return super(AnonUserProjectFilter, self)\ - .filter_queryset(request, queryset, view) + return super(AnonUserProjectFilter, self).filter_queryset( + request, queryset, view + ) class TagFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): # filter by tags if available. - tags = request.query_params.get('tags', None) + tags = request.query_params.get("tags", None) if tags and isinstance(tags, six.string_types): - tags = tags.split(',') + tags = tags.split(",") return queryset.filter(tags__name__in=tags) return queryset class XFormPermissionFilterMixin(object): - def _xform_filter(self, request, view, keyword): """Use XForm permissions""" - xform = request.query_params.get('xform') + xform = request.query_params.get("xform") public_forms = XForm.objects.none() if xform: - int_or_parse_error(xform, - u"Invalid value for formid. It must be a" - " positive integer.") + int_or_parse_error( + xform, "Invalid value for formid. It must be a" " positive integer." + ) self.xform = get_object_or_404(XForm, pk=xform) xform_qs = XForm.objects.filter(pk=self.xform.pk) - public_forms = XForm.objects.filter(pk=self.xform.pk, - shared_data=True) + public_forms = XForm.objects.filter(pk=self.xform.pk, shared_data=True) else: xform_qs = XForm.objects.all() xform_qs = xform_qs.filter(deleted_at=None) @@ -270,8 +284,12 @@ def _xform_filter(self, request, view, keyword): if request.user.is_anonymous: xforms = xform_qs.filter(shared_data=True) else: - xforms = super(XFormPermissionFilterMixin, self).filter_queryset( - request, xform_qs, view) | public_forms + xforms = ( + super(XFormPermissionFilterMixin, self).filter_queryset( + request, xform_qs, view + ) + | public_forms + ) return {"%s__in" % keyword: xforms} def _xform_filter_queryset(self, request, queryset, view, keyword): @@ -280,14 +298,14 @@ def _xform_filter_queryset(self, request, queryset, view, keyword): class ProjectPermissionFilterMixin(object): - def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") if project_id: - int_or_parse_error(project_id, - u"Invalid value for projectid. It must be a" - " positive integer.") + int_or_parse_error( + project_id, + "Invalid value for projectid. It must be a" " positive integer.", + ) project = get_object_or_404(Project, pk=project_id) project_qs = Project.objects.filter(pk=project.id) @@ -295,7 +313,8 @@ def _project_filter(self, request, view, keyword): project_qs = Project.objects.all() projects = super(ProjectPermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + request, project_qs, view + ) return {"%s__in" % keyword: projects} @@ -307,7 +326,6 @@ def _project_filter_queryset(self, request, queryset, view, keyword): class InstancePermissionFilterMixin(object): - def _instance_filter(self, request, view, keyword): instance_kwarg = {} instance_content_type = ContentType.objects.get_for_model(Instance) @@ -315,13 +333,14 @@ def _instance_filter(self, request, view, keyword): instance_id = request.query_params.get("instance") project_id = request.query_params.get("project") - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") if instance_id and project_id and xform_id: for object_id in [instance_id, project_id]: - int_or_parse_error(object_id, - u"Invalid value for instanceid. It must be" - " a positive integer.") + int_or_parse_error( + object_id, + "Invalid value for instanceid. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) # test if user has permissions on the project @@ -337,9 +356,9 @@ def _instance_filter(self, request, view, keyword): project_qs = Project.objects.filter(pk=project.id) if parent and parent.project == project: - projects = super( - InstancePermissionFilterMixin, self).filter_queryset( - request, project_qs, view) + projects = super(InstancePermissionFilterMixin, self).filter_queryset( + request, project_qs, view + ) instances = [instance.id] if projects else [] @@ -359,23 +378,21 @@ def _instance_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) -class RestServiceFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class RestServiceFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - return self._xform_filter_queryset( - request, queryset, view, 'xform_id') + return self._xform_filter_queryset(request, queryset, view, "xform_id") -class MetaDataFilter(ProjectPermissionFilterMixin, - InstancePermissionFilterMixin, - XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class MetaDataFilter( + ProjectPermissionFilterMixin, + InstancePermissionFilterMixin, + XFormPermissionFilterMixin, + ObjectPermissionsFilter, +): def filter_queryset(self, request, queryset, view): keyword = "object_id" - xform_id = request.query_params.get('xform') + xform_id = request.query_params.get("xform") project_id = request.query_params.get("project") instance_id = request.query_params.get("instance") @@ -392,8 +409,11 @@ def filter_queryset(self, request, queryset, view): # return instance specific metadata if instance_id: - return (queryset.filter(Q(**instance_kwarg)) - if (xform_id and instance_kwarg) else []) + return ( + queryset.filter(Q(**instance_kwarg)) + if (xform_id and instance_kwarg) + else [] + ) elif xform_id: # return xform specific metadata return queryset.filter(Q(**xform_kwarg)) @@ -403,31 +423,32 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(Q(**project_kwarg)) # return all project,instance and xform metadata information - return queryset.filter(Q(**xform_kwarg) | Q(**project_kwarg) | - Q(**instance_kwarg)) - + return queryset.filter( + Q(**xform_kwarg) | Q(**project_kwarg) | Q(**instance_kwarg) + ) -class AttachmentFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +class AttachmentFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - queryset = self._xform_filter_queryset(request, queryset, view, - 'instance__xform') + queryset = self._xform_filter_queryset( + request, queryset, view, "instance__xform" + ) # Ensure queryset is filtered by XForm meta permissions - xform_ids = set( - queryset.values_list("instance__xform", flat=True)) + xform_ids = set(queryset.values_list("instance__xform", flat=True)) for xform_id in xform_ids: xform = XForm.objects.get(id=xform_id) user = request.user queryset = exclude_items_from_queryset_using_xform_meta_perms( - xform, user, queryset) + xform, user, queryset + ) - instance_id = request.query_params.get('instance') + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer.") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer.", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -435,9 +456,8 @@ def filter_queryset(self, request, queryset, view): class AttachmentTypeFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - attachment_type = request.query_params.get('type') + attachment_type = request.query_params.get("type") mime_types = MEDIA_FILE_TYPES.get(attachment_type) @@ -448,15 +468,12 @@ def filter_queryset(self, request, queryset, view): class TeamOrgFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - org = request.data.get('org') or request.query_params.get('org') + org = request.data.get("org") or request.query_params.get("org") # Get all the teams for the organization if org: - kwargs = { - 'organization__username__iexact': org - } + kwargs = {"organization__username__iexact": org} return Team.objects.filter(**kwargs) @@ -464,26 +481,24 @@ def filter_queryset(self, request, queryset, view): class UserNoOrganizationsFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - if str(request.query_params.get('orgs')).lower() == 'false': + if str(request.query_params.get("orgs")).lower() == "false": organization_user_ids = OrganizationProfile.objects.values_list( - 'user__id', - flat=True) + "user__id", flat=True + ) queryset = queryset.exclude(id__in=organization_user_ids) return queryset class OrganizationsSharedWithUserFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): """ This returns a queryset containing only organizations to which the passed user belongs. """ - username = request.query_params.get('shared_with') + username = request.query_params.get("shared_with") if username: try: @@ -491,17 +506,14 @@ def filter_queryset(self, request, queryset, view): # Groups a User belongs to are available as a queryset property # of a User object, which this code takes advantage of - organization_user_ids = User.objects\ - .get(username=username)\ - .groups\ - .all()\ - .values_list( - 'team__organization', - flat=True)\ - .distinct() + organization_user_ids = ( + User.objects.get(username=username) + .groups.all() + .values_list("team__organization", flat=True) + .distinct() + ) - filtered_queryset = queryset.filter( - user_id__in=organization_user_ids) + filtered_queryset = queryset.filter(user_id__in=organization_user_ids) return filtered_queryset @@ -511,27 +523,22 @@ def filter_queryset(self, request, queryset, view): return queryset -class WidgetFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): - +class WidgetFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): - if view.action == 'list': + if view.action == "list": # Return widgets from xform user has perms to - return self._xform_filter_queryset(request, queryset, view, - 'object_id') + return self._xform_filter_queryset(request, queryset, view, "object_id") - return super(WidgetFilter, self).filter_queryset(request, queryset, - view) + return super(WidgetFilter, self).filter_queryset(request, queryset, view) class UserProfileFilter(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - if view.action == 'list': - users = request.GET.get('users') + if view.action == "list": + users = request.GET.get("users") if users: - users = users.split(',') + users = users.split(",") return queryset.filter(user__username__in=users) elif not request.user.is_anonymous: return queryset.filter(user__username=request.user.username) @@ -543,12 +550,13 @@ def filter_queryset(self, request, queryset, view): class NoteFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): - instance_id = request.query_params.get('instance') + instance_id = request.query_params.get("instance") if instance_id: - int_or_parse_error(instance_id, - u"Invalid value for instance_id. It must be" - " a positive integer") + int_or_parse_error( + instance_id, + "Invalid value for instance_id. It must be" " a positive integer", + ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -556,8 +564,7 @@ def filter_queryset(self, request, queryset, view): return queryset -class ExportFilter(XFormPermissionFilterMixin, - ObjectPermissionsFilter): +class ExportFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): """ ExportFilter class uses permissions on the related xform to filter Export queryesets. Also filters submitted_by a specific user. @@ -572,29 +579,30 @@ def _is_public_xform(self, export_id: int): return False def filter_queryset(self, request, queryset, view): - has_submitted_by_key = (Q(options__has_key='query') & - Q(options__query__has_key='_submitted_by'),) + has_submitted_by_key = ( + Q(options__has_key="query") & Q(options__query__has_key="_submitted_by"), + ) - if request.user.is_anonymous or self._is_public_xform( - view.kwargs.get('pk')): + if request.user.is_anonymous or self._is_public_xform(view.kwargs.get("pk")): return self._xform_filter_queryset( - request, queryset, view, 'xform_id')\ - .exclude(*has_submitted_by_key) + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) old_perm_format = self.perm_format # only if request.user has access to all data - self.perm_format = old_perm_format + '_all' - all_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .exclude(*has_submitted_by_key) + self.perm_format = old_perm_format + "_all" + all_qs = self._xform_filter_queryset( + request, queryset, view, "xform_id" + ).exclude(*has_submitted_by_key) # request.user has access to own submitted data - self.perm_format = old_perm_format + '_data' - submitter_qs = self._xform_filter_queryset(request, queryset, view, - 'xform_id')\ - .filter(*has_submitted_by_key)\ + self.perm_format = old_perm_format + "_data" + submitter_qs = ( + self._xform_filter_queryset(request, queryset, view, "xform_id") + .filter(*has_submitted_by_key) .filter(options__query___submitted_by=request.user.username) + ) return all_qs | submitter_qs diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index fe6711a66b..fd508a74d9 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -9,7 +9,9 @@ from typing import Tuple import pytz -from django.utils import six, timezone +import six + +from django.utils import timezone from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 9a155ad0fb..7850305d4b 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -6,6 +6,7 @@ import os import sys from datetime import datetime +import six import httplib2 from celery.backends.rpc import BacklogLimitExceeded @@ -13,14 +14,15 @@ from django.conf import settings from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils import six from django.utils.translation import ugettext as _ from kombu.exceptions import OperationalError from oauth2client import client as google_client -from oauth2client.client import (HttpAccessTokenRefreshError, - OAuth2WebServerFlow, TokenRevokeError) -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage +from oauth2client.client import ( + HttpAccessTokenRefreshError, + OAuth2WebServerFlow, + TokenRevokeError, +) +from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage from requests import ConnectionError from rest_framework import exceptions, status from rest_framework.response import Response @@ -30,43 +32,56 @@ from onadata.apps.main.models import TokenStorageModel from onadata.apps.viewer import tasks as viewer_task from onadata.apps.viewer.models.export import Export, ExportConnectionError -from onadata.libs.exceptions import (J2XException, NoRecordsFoundError, - NoRecordsPermission, ServiceUnavailable) +from onadata.libs.exceptions import ( + J2XException, + NoRecordsFoundError, + NoRecordsPermission, + ServiceUnavailable, +) from onadata.libs.permissions import filter_queryset_xform_meta_perms_sql from onadata.libs.utils import log -from onadata.libs.utils.async_status import (FAILED, PENDING, SUCCESSFUL, - async_status, - celery_state_to_status) -from onadata.libs.utils.common_tags import (DATAVIEW_EXPORT, - GROUPNAME_REMOVED_FLAG, OSM, - SUBMISSION_TIME) +from onadata.libs.utils.async_status import ( + FAILED, + PENDING, + SUCCESSFUL, + async_status, + celery_state_to_status, +) +from onadata.libs.utils.common_tags import ( + DATAVIEW_EXPORT, + GROUPNAME_REMOVED_FLAG, + OSM, + SUBMISSION_TIME, +) from onadata.libs.utils.common_tools import report_exception -from onadata.libs.utils.export_tools import (check_pending_export, - generate_attachments_zip_export, - generate_export, - generate_external_export, - generate_kml_export, - generate_osm_export, - newest_export_for, - parse_request_export_options, - should_create_new_export) +from onadata.libs.utils.export_tools import ( + check_pending_export, + generate_attachments_zip_export, + generate_export, + generate_external_export, + generate_kml_export, + generate_osm_export, + newest_export_for, + parse_request_export_options, + should_create_new_export, +) from onadata.libs.utils.logger_tools import response_with_mimetype_and_name from onadata.libs.utils.model_tools import get_columns_with_hxl # Supported external exports -EXTERNAL_EXPORT_TYPES = ['xls'] +EXTERNAL_EXPORT_TYPES = ["xls"] EXPORT_EXT = { - 'xls': Export.XLS_EXPORT, - 'xlsx': Export.XLS_EXPORT, - 'csv': Export.CSV_EXPORT, - 'csvzip': Export.CSV_ZIP_EXPORT, - 'savzip': Export.SAV_ZIP_EXPORT, - 'uuid': Export.EXTERNAL_EXPORT, - 'kml': Export.KML_EXPORT, - 'zip': Export.ZIP_EXPORT, + "xls": Export.XLS_EXPORT, + "xlsx": Export.XLS_EXPORT, + "csv": Export.CSV_EXPORT, + "csvzip": Export.CSV_ZIP_EXPORT, + "savzip": Export.SAV_ZIP_EXPORT, + "uuid": Export.EXTERNAL_EXPORT, + "kml": Export.KML_EXPORT, + "zip": Export.ZIP_EXPORT, OSM: Export.OSM_EXPORT, - 'gsheets': Export.GOOGLE_SHEETS_EXPORT + "gsheets": Export.GOOGLE_SHEETS_EXPORT, } @@ -89,53 +104,60 @@ def _get_export_type(export_type): export_type = EXPORT_EXT[export_type] else: raise exceptions.ParseError( - _(u"'%(export_type)s' format not known or not implemented!" % - {'export_type': export_type})) + _( + "'%(export_type)s' format not known or not implemented!" + % {"export_type": export_type} + ) + ) return export_type # pylint: disable=too-many-arguments, too-many-locals, too-many-branches -def custom_response_handler(request, - xform, - query, - export_type, - token=None, - meta=None, - dataview=False, - filename=None): +def custom_response_handler( + request, + xform, + query, + export_type, + token=None, + meta=None, + dataview=False, + filename=None, +): """ Returns a HTTP response with export file for download. """ export_type = _get_export_type(export_type) - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT options = parse_request_export_options(request.query_params) - dataview_pk = hasattr(dataview, 'pk') and dataview.pk + dataview_pk = hasattr(dataview, "pk") and dataview.pk options["dataview_pk"] = dataview_pk if dataview: - columns_with_hxl = get_columns_with_hxl(xform.survey.get('children')) + columns_with_hxl = get_columns_with_hxl(xform.survey.get("children")) if columns_with_hxl: - options['include_hxl'] = include_hxl_row(dataview.columns, - list(columns_with_hxl)) + options["include_hxl"] = include_hxl_row( + dataview.columns, list(columns_with_hxl) + ) try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: return Response( - data=json.dumps({ - "details": _("You don't have permission") - }), + data=json.dumps({"details": _("You don't have permission")}), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) if query: - options['query'] = query + options["query"] = query remove_group_name = options.get("remove_group_name") @@ -147,20 +169,21 @@ def custom_response_handler(request, if export_type == Export.GOOGLE_SHEETS_EXPORT: return Response( - data=json.dumps({ - "details": _("Sheets export only supported in async mode") - }), + data=json.dumps( + {"details": _("Sheets export only supported in async mode")} + ), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) # check if we need to re-generate, # we always re-generate if a filter is specified def _new_export(): return _generate_new_export( - request, xform, query, export_type, dataview_pk=dataview_pk) + request, xform, query, export_type, dataview_pk=dataview_pk + ) - if should_create_new_export( - xform, export_type, options, request=request): + if should_create_new_export(xform, export_type, options, request=request): export = _new_export() else: export = newest_export_for(xform, export_type, options) @@ -184,7 +207,8 @@ def _new_export(): show_date = True if filename is None and export.status == Export.SUCCESSFUL: filename = _generate_filename( - request, xform, remove_group_name, dataview_pk=dataview_pk) + request, xform, remove_group_name, dataview_pk=dataview_pk + ) else: show_date = False response = response_with_mimetype_and_name( @@ -192,13 +216,13 @@ def _new_export(): filename, extension=ext, show_date=show_date, - file_path=export.filepath) + file_path=export.filepath, + ) return response -def _generate_new_export(request, xform, query, export_type, - dataview_pk=False): +def _generate_new_export(request, xform, query, export_type, dataview_pk=False): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) @@ -208,18 +232,17 @@ def _generate_new_export(request, xform, query, export_type, "id_string": xform.id_string, } if query: - options['query'] = query + options["query"] = query options["dataview_pk"] = dataview_pk if export_type == Export.GOOGLE_SHEETS_EXPORT: - options['google_credentials'] = \ - _get_google_credential(request).to_json() + options["google_credentials"] = _get_google_credential(request).to_json() try: if export_type == Export.EXTERNAL_EXPORT: - options['token'] = request.GET.get('token') - options['data_id'] = request.GET.get('data_id') - options['meta'] = request.GET.get('meta') + options["token"] = request.GET.get("token") + options["data_id"] = request.GET.get("data_id") + options["meta"] = request.GET.get("meta") export = generate_external_export( export_type, @@ -227,7 +250,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, @@ -235,7 +259,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.ZIP_EXPORT: export = generate_attachments_zip_export( export_type, @@ -243,7 +268,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, @@ -251,7 +277,8 @@ def _generate_new_export(request, xform, query, export_type, xform.id_string, None, options, - xform=xform) + xform=xform, + ) else: options.update(parse_request_export_options(request.query_params)) @@ -259,10 +286,14 @@ def _generate_new_export(request, xform, query, export_type, audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_CREATED, request.user, xform.user, - _("Created %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + log.Actions.EXPORT_CREATED, + request.user, + xform.user, + _("Created %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: @@ -281,10 +312,14 @@ def log_export(request, xform, export_type): # log download as well audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( - log.Actions.EXPORT_DOWNLOADED, request.user, xform.user, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + log.Actions.EXPORT_DOWNLOADED, + request.user, + xform.user, + _("Downloaded %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) def external_export_response(export): @@ -292,21 +327,16 @@ def external_export_response(export): Redirects to export_url of XLSReports successful export. In case of a failure, returns a 400 HTTP JSON response with the error message. """ - if isinstance(export, Export) \ - and export.internal_status == Export.SUCCESSFUL: + if isinstance(export, Export) and export.internal_status == Export.SUCCESSFUL: return HttpResponseRedirect(export.export_url) else: http_status = status.HTTP_400_BAD_REQUEST - return Response( - json.dumps(export), http_status, content_type="application/json") + return Response(json.dumps(export), http_status, content_type="application/json") -def _generate_filename(request, - xform, - remove_group_name=False, - dataview_pk=False): - if request.GET.get('raw'): +def _generate_filename(request, xform, remove_group_name=False, dataview_pk=False): + if request.GET.get("raw"): filename = None else: # append group name removed flag otherwise use the form id_string @@ -322,22 +352,24 @@ def _generate_filename(request, def _set_start_end_params(request, query): # check for start and end params - if 'start' in request.GET or 'end' in request.GET: - query = json.loads(query) \ - if isinstance(query, six.string_types) else query + if "start" in request.GET or "end" in request.GET: + query = json.loads(query) if isinstance(query, six.string_types) else query query[SUBMISSION_TIME] = {} try: - if request.GET.get('start'): - query[SUBMISSION_TIME]['$gte'] = _format_date_for_mongo( - request.GET['start'], datetime) - - if request.GET.get('end'): - query[SUBMISSION_TIME]['$lte'] = _format_date_for_mongo( - request.GET['end'], datetime) + if request.GET.get("start"): + query[SUBMISSION_TIME]["$gte"] = _format_date_for_mongo( + request.GET["start"], datetime + ) + + if request.GET.get("end"): + query[SUBMISSION_TIME]["$lte"] = _format_date_for_mongo( + request.GET["end"], datetime + ) except ValueError: raise exceptions.ParseError( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) else: query = json.dumps(query) @@ -348,16 +380,15 @@ def _get_extension_from_export_type(export_type): extension = export_type if export_type == Export.XLS_EXPORT: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" return extension def _format_date_for_mongo(x, datetime): # pylint: disable=W0621, C0103 - return datetime.strptime(x, - '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S') + return datetime.strptime(x, "%y_%m_%d_%H_%M_%S").strftime("%Y-%m-%dT%H:%M:%S") def process_async_export(request, xform, export_type, options=None): @@ -388,20 +419,23 @@ def process_async_export(request, xform, export_type, options=None): force_xlsx = options.get("force_xlsx") try: - query = filter_queryset_xform_meta_perms_sql(xform, request.user, - query) + query = filter_queryset_xform_meta_perms_sql(xform, request.user, query) except NoRecordsPermission: payload = {"details": _("You don't have permission")} return Response( data=json.dumps(payload), status=status.HTTP_403_FORBIDDEN, - content_type="application/json") + content_type="application/json", + ) else: if query: - options['query'] = query + options["query"] = query - if export_type in EXTERNAL_EXPORT_TYPES and \ - (token is not None) or (meta is not None): + if ( + export_type in EXTERNAL_EXPORT_TYPES + and (token is not None) + or (meta is not None) + ): export_type = Export.EXTERNAL_EXPORT if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -409,25 +443,27 @@ def process_async_export(request, xform, export_type, options=None): if isinstance(credential, HttpResponseRedirect): return credential - options['google_credentials'] = credential.to_json() + options["google_credentials"] = credential.to_json() - if should_create_new_export(xform, export_type, options, request=request)\ - or export_type == Export.EXTERNAL_EXPORT: + if ( + should_create_new_export(xform, export_type, options, request=request) + or export_type == Export.EXTERNAL_EXPORT + ): resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, force_xlsx, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, force_xlsx, options=options + ) } else: - print('Do not create a new export.') + print("Do not create a new export.") export = newest_export_for(xform, export_type, options) if not export.filename: # tends to happen when using newest_export_for. resp = { - u'job_uuid': - _create_export_async( - xform, export_type, query, force_xlsx, options=options) + "job_uuid": _create_export_async( + xform, export_type, query, force_xlsx, options=options + ) } else: resp = export_async_export_response(request, export) @@ -435,21 +471,19 @@ def process_async_export(request, xform, export_type, options=None): return resp -def _create_export_async(xform, - export_type, - query=None, - force_xlsx=False, - options=None): +def _create_export_async( + xform, export_type, query=None, force_xlsx=False, options=None +): + """ + Creates async exports + :param xform: + :param export_type: + :param query: + :param force_xlsx: + :param options: + :return: + job_uuid generated """ - Creates async exports - :param xform: - :param export_type: - :param query: - :param force_xlsx: - :param options: - :return: - job_uuid generated - """ export = check_pending_export(xform, export_type, options) if export: @@ -457,7 +491,8 @@ def _create_export_async(xform, try: export, async_result = viewer_task.create_async_export( - xform, export_type, query, force_xlsx, options=options) + xform, export_type, query, force_xlsx, options=options + ) except ExportConnectionError: raise ServiceUnavailable @@ -473,14 +508,16 @@ def export_async_export_response(request, export): """ if export.status == Export.SUCCESSFUL: if export.export_type not in [ - Export.EXTERNAL_EXPORT, Export.GOOGLE_SHEETS_EXPORT + Export.EXTERNAL_EXPORT, + Export.GOOGLE_SHEETS_EXPORT, ]: export_url = reverse( - 'export-detail', kwargs={'pk': export.pk}, request=request) + "export-detail", kwargs={"pk": export.pk}, request=request + ) else: export_url = export.export_url resp = async_status(SUCCESSFUL) - resp['export_url'] = export_url + resp["export_url"] = export_url elif export.status == Export.PENDING: resp = async_status(PENDING) else: @@ -493,13 +530,14 @@ def get_async_response(job_uuid, request, xform, count=0): """ Returns the status of an async task for the given job_uuid. """ + def _get_response(): export = get_object_or_404(Export, task_id=job_uuid) return export_async_export_response(request, export) try: job = AsyncResult(job_uuid) - if job.state == 'SUCCESS': + if job.state == "SUCCESS": resp = _get_response() else: resp = async_status(celery_state_to_status(job.state)) @@ -510,7 +548,7 @@ def _get_response(): if isinstance(result, dict): resp.update(result) else: - resp.update({'progress': str(result)}) + resp.update({"progress": str(result)}) except (OperationalError, ConnectionError) as e: report_exception("Connection Error", e, sys.exc_info()) if count > 0: @@ -519,7 +557,7 @@ def _get_response(): return get_async_response(job_uuid, request, xform, count + 1) except BacklogLimitExceeded: # most likely still processing - resp = async_status(celery_state_to_status('PENDING')) + resp = async_status(celery_state_to_status("PENDING")) return resp @@ -528,9 +566,9 @@ def response_for_format(data, format=None): # pylint: disable=W0622 """ Return appropriately formatted data in Response(). """ - if format == 'xml': + if format == "xml": formatted_data = data.xml - elif format == 'xls': + elif format == "xls": if not data.xls or not data.xls.storage.exists(data.xls.name): raise Http404() @@ -544,35 +582,38 @@ def generate_google_web_flow(request): """ Returns a OAuth2WebServerFlow object from the request redirect_uri. """ - if 'redirect_uri' in request.GET: - redirect_uri = request.GET.get('redirect_uri') - elif 'redirect_uri' in request.POST: - redirect_uri = request.POST.get('redirect_uri') - elif 'redirect_uri' in request.query_params: - redirect_uri = request.query_params.get('redirect_uri') - elif 'redirect_uri' in request.data: - redirect_uri = request.data.get('redirect_uri') + if "redirect_uri" in request.GET: + redirect_uri = request.GET.get("redirect_uri") + elif "redirect_uri" in request.POST: + redirect_uri = request.POST.get("redirect_uri") + elif "redirect_uri" in request.query_params: + redirect_uri = request.query_params.get("redirect_uri") + elif "redirect_uri" in request.data: + redirect_uri = request.data.get("redirect_uri") else: redirect_uri = settings.GOOGLE_STEP2_URI return OAuth2WebServerFlow( client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=' '.join([ - 'https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', - 'https://www.googleapis.com/auth/drive.file' - ]), + scope=" ".join( + [ + "https://docs.google.com/feeds/", + "https://spreadsheets.google.com/feeds/", + "https://www.googleapis.com/auth/drive.file", + ] + ), redirect_uri=redirect_uri, - prompt="consent") + prompt="consent", + ) def _get_google_credential(request): token = None credential = None if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') + storage = Storage(TokenStorageModel, "id", request.user, "credential") credential = storage.get() - elif request.session.get('access_token'): + elif request.session.get("access_token"): credential = google_client.OAuth2Credentials.from_json(token) if credential: diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 05a087b28b..535b7d466e 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -191,7 +191,7 @@ "django.contrib.admindocs", "django.contrib.gis", "registration", - "django_nose", + # "django_nose", "django_digest", "corsheaders", "oauth2_provider", @@ -462,8 +462,8 @@ def configure_logging(logger, **kwargs): # default content length for submission requests DEFAULT_CONTENT_LENGTH = 10000000 -TEST_RUNNER = "django_nose.NoseTestSuiteRunner" -NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] +# TEST_RUNNER = "django_nose.NoseTestSuiteRunner" +# NOSE_ARGS = ["--with-fixture-bundling", "--nologcapture", "--nocapture"] # fake endpoints for testing TEST_HTTP_HOST = "testserver.com" From 357688504f8656492b0de1d0719e787f965815f3 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:41:53 +0300 Subject: [PATCH 012/234] Disable SPSS export in the event of an ImportError exception. savReaderWriter needs to be updated to handle collections Iterable correctly. --- onadata/libs/utils/api_export_tools.py | 6 +++++- onadata/libs/utils/export_builder.py | 10 +++++++++- onadata/libs/utils/export_tools.py | 7 ++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 7850305d4b..7becf8c34a 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -27,7 +27,11 @@ from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse -from savReaderWriter import SPSSIOError + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.main.models import TokenStorageModel from onadata.apps.viewer import tasks as viewer_task diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 018b7f74ec..c54483a5e0 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -23,7 +23,11 @@ from openpyxl.workbook import Workbook from pyxform.question import Question from pyxform.section import RepeatingSection, Section -from savReaderWriter import SavWriter + +try: + from savReaderWriter import SavWriter +except ImportError: + SavWriter = None from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.logger.models.xform import ( @@ -1411,6 +1415,10 @@ def _check_sav_column(self, column, columns): return column def to_zipped_sav(self, path, data, *args, **kwargs): + if SavWriter is None: + # Fail silently + return + total_records = kwargs.get("total_records") def write_row(row, csv_writer, fields): diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index f4ba38bd7e..dc12d5ddee 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -26,7 +26,12 @@ from six import iteritems from json2xlsclient.client import Client from rest_framework import exceptions -from savReaderWriter import SPSSIOError + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception + from multidb.pinning import use_master from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm From 4884d9d51c0ee410f517980e40435ad194eb340b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:48:46 +0300 Subject: [PATCH 013/234] django.utils.decorators.available_attrs() - This function returns functools.WRAPPER_ASSIGNMENTS Removed in Django 3.0 --- onadata/libs/renderers/renderers.py | 2 +- onadata/libs/utils/decorators.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index fd508a74d9..077c189135 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -15,7 +15,7 @@ from django.utils.dateparse import parse_datetime from django.utils.encoding import smart_text, force_str from django.utils.xmlutils import SimplerXMLGenerator -from fsix import iteritems +from six import iteritems from rest_framework import negotiation from rest_framework.renderers import ( BaseRenderer, diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 0d556d8efe..55a0f6cc46 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -2,7 +2,6 @@ from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME -from django.utils.decorators import available_attrs from django.conf import settings from django.http import HttpResponseRedirect @@ -17,7 +16,7 @@ def with_check_obj(*args, **kwargs): def is_owner(view_func): - @wraps(view_func, assigned=available_attrs(view_func)) + @wraps(view_func) def _wrapped_view(request, *args, **kwargs): # assume username is first arg if request.user.is_authenticated: From 4b0333e7e6a6de5108f86b89e7d50a8d39de01db Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Apr 2022 05:52:20 +0300 Subject: [PATCH 014/234] Disable SPSS export in the event of an ImportError exception. savReaderWriter needs to be updated to handle collections Iterable correctly. --- onadata/apps/viewer/views.py | 760 ++++++++++++++++++++--------------- 1 file changed, 441 insertions(+), 319 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 03e24606a4..b5d260846d 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -16,9 +16,13 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage, get_storage_class -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseNotFound, - HttpResponseRedirect) +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, +) from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse @@ -26,9 +30,12 @@ from django.views.decorators.http import require_POST from dpath import util as dpath_util from oauth2client import client as google_client -from oauth2client.contrib.django_util.storage import \ - DjangoORMStorage as Storage -from savReaderWriter import SPSSIOError +from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage + +try: + from savReaderWriter import SPSSIOError +except ImportError: + SPSSIOError = Exception from onadata.apps.logger.models import Attachment from onadata.apps.logger.views import download_jsonform @@ -40,17 +47,30 @@ from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.chart_tools import build_chart_data from onadata.libs.utils.export_tools import ( - DEFAULT_GROUP_DELIMITER, generate_export, kml_export_data, - newest_export_for, should_create_new_export, str_to_bool) + DEFAULT_GROUP_DELIMITER, + generate_export, + kml_export_data, + newest_export_for, + should_create_new_export, + str_to_bool, +) from onadata.libs.utils.google import google_flow from onadata.libs.utils.image_tools import image_url from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import ( - generate_content_disposition_header, response_with_mimetype_and_name) -from onadata.libs.utils.user_auth import (get_xform_and_perms, has_permission, - helper_auth_helper) + generate_content_disposition_header, + response_with_mimetype_and_name, +) +from onadata.libs.utils.user_auth import ( + get_xform_and_perms, + has_permission, + helper_auth_helper, +) from onadata.libs.utils.viewer_tools import ( - create_attachments_zipfile, export_def_from_filename, get_form) + create_attachments_zipfile, + export_def_from_filename, + get_form, +) from onadata.libs.utils.common_tools import get_uuid @@ -58,15 +78,18 @@ def _get_start_end_submission_time(request): start = None end = None try: - if request.GET.get('start'): - start = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['start'], '%y_%m_%d_%H_%M_%S')) - if request.GET.get('end'): - end = pytz.timezone('UTC').localize( - datetime.strptime(request.GET['end'], '%y_%m_%d_%H_%M_%S')) + if request.GET.get("start"): + start = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["start"], "%y_%m_%d_%H_%M_%S") + ) + if request.GET.get("end"): + end = pytz.timezone("UTC").localize( + datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S") + ) except ValueError: return HttpResponseBadRequest( - _("Dates must be in the format YY_MM_DD_hh_mm_ss")) + _("Dates must be in the format YY_MM_DD_hh_mm_ss") + ) return start, end @@ -75,16 +98,16 @@ def encode(time_str): """ Reformat a time string into YYYY-MM-dd HH:mm:ss. """ - return strftime("%Y-%m-%d %H:%M:%S", - strptime(time_str, "%Y_%m_%d_%H_%M_%S")) + return strftime("%Y-%m-%d %H:%M:%S", strptime(time_str, "%Y_%m_%d_%H_%M_%S")) def format_date_for_mongo(time_str): """ Reformat a time string into YYYY-MM-ddTHH:mm:ss. """ - return datetime.strptime(time_str, '%y_%m_%d_%H_%M_%S')\ - .strftime('%Y-%m-%dT%H:%M:%S') + return datetime.strptime(time_str, "%y_%m_%d_%H_%M_%S").strftime( + "%Y-%m-%dT%H:%M:%S" + ) def instances_for_export(data_dictionary, start=None, end=None): @@ -93,9 +116,9 @@ def instances_for_export(data_dictionary, start=None, end=None): """ kwargs = dict() if start: - kwargs['date_created__gte'] = start + kwargs["date_created__gte"] = start if end: - kwargs['date_created__lte'] = end + kwargs["date_created__lte"] = end return data_dictionary.instances.filter(**kwargs) @@ -108,11 +131,9 @@ def set_instances_for_export(id_string, owner, request): is successful or not respectively. """ data_dictionary = get_object_or_404( - DataDictionary, - id_string__iexact=id_string, - user=owner, - deletd_at__isnull=True) - start, end = request.GET.get('start'), request.GET.get('end') + DataDictionary, id_string__iexact=id_string, user=owner, deletd_at__isnull=True + ) + start, end = request.GET.get("start"), request.GET.get("end") if start: try: start = encode(start) @@ -121,7 +142,8 @@ def set_instances_for_export(id_string, owner, request): return [ False, HttpResponseBadRequest( - _(u'Start time format must be YY_MM_DD_hh_mm_ss')) + _("Start time format must be YY_MM_DD_hh_mm_ss") + ), ] if end: try: @@ -130,12 +152,12 @@ def set_instances_for_export(id_string, owner, request): # bad format return [ False, - HttpResponseBadRequest( - _(u'End time format must be YY_MM_DD_hh_mm_ss')) + HttpResponseBadRequest(_("End time format must be YY_MM_DD_hh_mm_ss")), ] if start or end: data_dictionary.instances_for_export = instances_for_export( - data_dictionary, start, end) + data_dictionary, start, end + ) return [True, data_dictionary] @@ -147,48 +169,48 @@ def average(values): return sum(values, 0.0) / len(values) if values else None -def map_view(request, username, id_string, template='map.html'): +def map_view(request, username, id_string, template="map.html"): """ Map view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'content_user': owner, 'xform': xform} - data['profile'], __ = UserProfile.objects.get_or_create(user=owner) - - data['form_view'] = True - data['jsonform_url'] = reverse( - download_jsonform, - kwargs={"username": username, - "id_string": id_string}) - data['enketo_edit_url'] = reverse( - 'edit_data', - kwargs={"username": username, - "id_string": id_string, - "data_id": 0}) - data['enketo_add_url'] = reverse( - 'enter_data', kwargs={"username": username, - "id_string": id_string}) - - data['enketo_add_with_url'] = reverse( - 'add_submission_with', - kwargs={"username": username, - "id_string": id_string}) - data['mongo_api_url'] = reverse( - 'mongo_view_api', - kwargs={"username": username, - "id_string": id_string}) - data['delete_data_url'] = reverse( - 'delete_data', kwargs={"username": username, - "id_string": id_string}) - data['mapbox_layer'] = MetaData.mapbox_layer_upload(xform) + return HttpResponseForbidden(_("Not shared.")) + data = {"content_user": owner, "xform": xform} + data["profile"], __ = UserProfile.objects.get_or_create(user=owner) + + data["form_view"] = True + data["jsonform_url"] = reverse( + download_jsonform, kwargs={"username": username, "id_string": id_string} + ) + data["enketo_edit_url"] = reverse( + "edit_data", kwargs={"username": username, "id_string": id_string, "data_id": 0} + ) + data["enketo_add_url"] = reverse( + "enter_data", kwargs={"username": username, "id_string": id_string} + ) + + data["enketo_add_with_url"] = reverse( + "add_submission_with", kwargs={"username": username, "id_string": id_string} + ) + data["mongo_api_url"] = reverse( + "mongo_view_api", kwargs={"username": username, "id_string": id_string} + ) + data["delete_data_url"] = reverse( + "delete_data", kwargs={"username": username, "id_string": id_string} + ) + data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) audit = {"xform": xform.id_string} - audit_log(Actions.FORM_MAP_VIEWED, request.user, owner, - _("Requested map on '%(id_string)s'.") % - {'id_string': xform.id_string}, audit, request) + audit_log( + Actions.FORM_MAP_VIEWED, + request.user, + owner, + _("Requested map on '%(id_string)s'.") % {"id_string": xform.id_string}, + audit, + request, + ) return render(request, template, data) @@ -196,7 +218,7 @@ def map_embed_view(request, username, id_string): """ Embeded map view. """ - return map_view(request, username, id_string, template='map_embed.html') + return map_view(request, username, id_string, template="map_embed.html") def add_submission_with(request, username, id_string): @@ -209,51 +231,51 @@ def geopoint_xpaths(username, id_string): Returns xpaths with elements of type 'geopoint'. """ data_dictionary = DataDictionary.objects.get( - user__username__iexact=username, id_string__iexact=id_string) + user__username__iexact=username, id_string__iexact=id_string + ) return [ e.get_abbreviated_xpath() for e in data_dictionary.get_survey_elements() - if e.bind.get(u'type') == u'geopoint' + if e.bind.get("type") == "geopoint" ] - value = request.GET.get('coordinates') + value = request.GET.get("coordinates") xpaths = geopoint_xpaths(username, id_string) xml_dict = {} for path in xpaths: dpath_util.new(xml_dict, path, value) context = { - 'username': username, - 'id_string': id_string, - 'xml_content': dict2xml(xml_dict) + "username": username, + "id_string": id_string, + "xml_content": dict2xml(xml_dict), } - instance_xml = loader.get_template("instance_add.xml")\ - .render(context) + instance_xml = loader.get_template("instance_add.xml").render(context) url = settings.ENKETO_API_INSTANCE_IFRAME_URL return_url = reverse( - 'thank_you_submission', - kwargs={"username": username, - "id_string": id_string}) + "thank_you_submission", kwargs={"username": username, "id_string": id_string} + ) if settings.DEBUG: openrosa_url = "https://dev.formhub.org/{}".format(username) else: openrosa_url = request.build_absolute_uri("/{}".format(username)) payload = { - 'return_url': return_url, - 'form_id': id_string, - 'server_url': openrosa_url, - 'instance': instance_xml, - 'instance_id': get_uuid() + "return_url": return_url, + "form_id": id_string, + "server_url": openrosa_url, + "instance": instance_xml, + "instance_id": get_uuid(), } response = requests.post( url, data=payload, - auth=(settings.ENKETO_API_TOKEN, ''), - verify=getattr(settings, 'VERIFY_SSL', True)) + auth=(settings.ENKETO_API_TOKEN, ""), + verify=getattr(settings, "VERIFY_SSL", True), + ) - return HttpResponse(response.text, content_type='application/json') + return HttpResponse(response.text, content_type="application/json") # pylint: disable=W0613 @@ -270,35 +292,35 @@ def data_export(request, username, id_string, export_type): Data export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) query = request.GET.get("query") extension = export_type # check if we should force xlsx - force_xlsx = request.GET.get('xls') != 'true' + force_xlsx = request.GET.get("xls") != "true" if export_type == Export.XLS_EXPORT and force_xlsx: - extension = 'xlsx' + extension = "xlsx" elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' + extension = "zip" audit = {"xform": xform.id_string, "export_type": export_type} - options = { - "extension": extension, - "username": username, - "id_string": id_string - } + options = {"extension": extension, "username": username, "id_string": id_string} if query: - options['query'] = query + options["query"] = query # check if we need to re-generate, # we always re-generate if a filter is specified - if should_create_new_export(xform, export_type, options) or query or\ - 'start' in request.GET or 'end' in request.GET: + if ( + should_create_new_export(xform, export_type, options) + or query + or "start" in request.GET + or "end" in request.GET + ): # check for start and end params start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) @@ -306,11 +328,14 @@ def data_export(request, username, id_string, export_type): try: export = generate_export(export_type, xform, None, options) audit_log( - Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - 'export_type': export_type.upper() - }, audit, request) + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) except NoRecordsFoundError: return HttpResponseNotFound(_("No records found to export")) except SPSSIOError as e: @@ -320,10 +345,14 @@ def data_export(request, username, id_string, export_type): # log download as well audit_log( - Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export on '%(id_string)s'.") % - {'id_string': xform.id_string, - 'export_type': export_type.upper()}, audit, request) + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded %(export_type)s export on '%(id_string)s'.") + % {"id_string": xform.id_string, "export_type": export_type.upper()}, + audit, + request, + ) if not export.filename and not export.error_message: # tends to happen when using newset_export_for. @@ -335,14 +364,15 @@ def data_export(request, username, id_string, export_type): # xlsx if it exceeds limits __, extension = os.path.splitext(export.filename) extension = extension[1:] - if request.GET.get('raw'): + if request.GET.get("raw"): id_string = None response = response_with_mimetype_and_name( Export.EXPORT_MIMES[extension], id_string, extension=extension, - file_path=export.filepath) + file_path=export.filepath, + ) return response @@ -355,15 +385,15 @@ def create_export(request, username, id_string, export_type): Create async export tasks view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) credential = None if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -373,69 +403,78 @@ def create_export(request, username, id_string, export_type): return credential query = request.POST.get("query") - force_xlsx = request.POST.get('xls') != 'true' + force_xlsx = request.POST.get("xls") != "true" # export options - group_delimiter = request.POST.get("options[group_delimiter]", '/') - if group_delimiter not in ['.', '/']: + group_delimiter = request.POST.get("options[group_delimiter]", "/") + if group_delimiter not in [".", "/"]: return HttpResponseBadRequest( - _("%s is not a valid delimiter" % group_delimiter)) + _("%s is not a valid delimiter" % group_delimiter) + ) # default is True, so when dont_.. is yes # split_select_multiples becomes False - split_select_multiples = request.POST.get( - "options[dont_split_select_multiples]", "no") == "no" + split_select_multiples = ( + request.POST.get("options[dont_split_select_multiples]", "no") == "no" + ) - binary_select_multiples = getattr(settings, 'BINARY_SELECT_MULTIPLES', - False) + binary_select_multiples = getattr(settings, "BINARY_SELECT_MULTIPLES", False) remove_group_name = request.POST.get("options[remove_group_name]", "false") value_select_multiples = request.POST.get( - "options[value_select_multiples]", "false") + "options[value_select_multiples]", "false" + ) # external export option meta = request.POST.get("meta") options = { - 'group_delimiter': group_delimiter, - 'split_select_multiples': split_select_multiples, - 'binary_select_multiples': binary_select_multiples, - 'value_select_multiples': str_to_bool(value_select_multiples), - 'remove_group_name': str_to_bool(remove_group_name), - 'meta': meta.replace(",", "") if meta else None, - 'google_credentials': credential + "group_delimiter": group_delimiter, + "split_select_multiples": split_select_multiples, + "binary_select_multiples": binary_select_multiples, + "value_select_multiples": str_to_bool(value_select_multiples), + "remove_group_name": str_to_bool(remove_group_name), + "meta": meta.replace(",", "") if meta else None, + "google_credentials": credential, } try: create_async_export(xform, export_type, query, force_xlsx, options) except ExportTypeError: - return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + return HttpResponseBadRequest(_("%s is not a valid export type" % export_type)) else: audit = {"xform": xform.id_string, "export_type": export_type} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") % { - 'export_type': export_type.upper(), - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created %(export_type)s export on '%(id_string)s'.") + % { + "export_type": export_type.upper(), + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def _get_google_credential(request): token = None if request.user.is_authenticated: - storage = Storage(TokenStorageModel, 'id', request.user, 'credential') + storage = Storage(TokenStorageModel, "id", request.user, "credential") credential = storage.get() - elif request.session.get('access_token'): + elif request.session.get("access_token"): credential = google_client.OAuth2Credentials.from_json(token) - return credential or HttpResponseRedirect( - google_flow.step1_get_authorize_url()) + return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) def export_list(request, username, id_string, export_type): @@ -446,7 +485,8 @@ def export_list(request, username, id_string, export_type): if export_type not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( - _(u'Export type "%s" is not supported.' % export_type)) + _('Export type "%s" is not supported.' % export_type) + ) if export_type == Export.GOOGLE_SHEETS_EXPORT: # Retrieve google creds or redirect to google authorization page @@ -454,58 +494,57 @@ def export_list(request, username, id_string, export_type): if isinstance(credential, HttpResponseRedirect): return credential owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) if export_type == Export.EXTERNAL_EXPORT: # check for template before trying to generate a report if not MetaData.external_export(xform): - return HttpResponseForbidden(_(u'No XLS Template set.')) + return HttpResponseForbidden(_("No XLS Template set.")) # Get meta and token - export_token = request.GET.get('token') - export_meta = request.GET.get('meta') + export_token = request.GET.get("token") + export_meta = request.GET.get("meta") options = { - 'group_delimiter': DEFAULT_GROUP_DELIMITER, - 'remove_group_name': False, - 'split_select_multiples': True, - 'binary_select_multiples': False, - 'meta': export_meta, - 'token': export_token, - 'google_credentials': credential + "group_delimiter": DEFAULT_GROUP_DELIMITER, + "remove_group_name": False, + "split_select_multiples": True, + "binary_select_multiples": False, + "meta": export_meta, + "token": export_token, + "google_credentials": credential, } if should_create_new_export(xform, export_type, options): try: create_async_export( - xform, - export_type, - query=None, - force_xlsx=True, - options=options) + xform, export_type, query=None, force_xlsx=True, options=options + ) except Export.ExportTypeError: return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type)) + _("%s is not a valid export type" % export_type) + ) - metadata_qs = MetaData.objects.filter(object_id=xform.id, - data_type="external_export")\ - .values('id', 'data_value') + metadata_qs = MetaData.objects.filter( + object_id=xform.id, data_type="external_export" + ).values("id", "data_value") for metadata in metadata_qs: - metadata['data_value'] = metadata.get('data_value').split('|')[0] + metadata["data_value"] = metadata.get("data_value").split("|")[0] data = { - 'username': owner.username, - 'xform': xform, - 'export_type': export_type, - 'export_type_name': Export.EXPORT_TYPE_DICT[export_type], - 'exports': Export.objects.filter( - xform=xform, export_type=export_type).order_by('-created_on'), - 'metas': metadata_qs + "username": owner.username, + "xform": xform, + "export_type": export_type, + "export_type_name": Export.EXPORT_TYPE_DICT[export_type], + "exports": Export.objects.filter(xform=xform, export_type=export_type).order_by( + "-created_on" + ), + "metas": metadata_qs, } # yapf: disable - return render(request, 'export_list.html', data) + return render(request, "export_list.html", data) def export_progress(request, username, id_string, export_type): @@ -513,47 +552,52 @@ def export_progress(request, username, id_string, export_type): Async export progress view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db - export_ids = request.GET.getlist('export_ids') + export_ids = request.GET.getlist("export_ids") exports = Export.objects.filter( - xform=xform, id__in=export_ids, export_type=export_type) + xform=xform, id__in=export_ids, export_type=export_type + ) statuses = [] for export in exports: status = { - 'complete': False, - 'url': None, - 'filename': None, - 'export_id': export.id + "complete": False, + "url": None, + "filename": None, + "export_id": export.id, } if export.status == Export.SUCCESSFUL: - status['url'] = reverse( + status["url"] = reverse( export_download, kwargs={ - 'username': owner.username, - 'id_string': xform.id_string, - 'export_type': export.export_type, - 'filename': export.filename - }) - status['filename'] = export.filename - if export.export_type == Export.GOOGLE_SHEETS_EXPORT and \ - export.export_url is None: - status['url'] = None - if export.export_type == Export.EXTERNAL_EXPORT \ - and export.export_url is None: - status['url'] = None + "username": owner.username, + "id_string": xform.id_string, + "export_type": export.export_type, + "filename": export.filename, + }, + ) + status["filename"] = export.filename + if ( + export.export_type == Export.GOOGLE_SHEETS_EXPORT + and export.export_url is None + ): + status["url"] = None + if ( + export.export_type == Export.EXTERNAL_EXPORT + and export.export_url is None + ): + status["url"] = None # mark as complete if it either failed or succeeded but NOT pending - if export.status == Export.SUCCESSFUL \ - or export.status == Export.FAILED: - status['complete'] = True + if export.status == Export.SUCCESSFUL or export.status == Export.FAILED: + status["complete"] = True statuses.append(status) - return HttpResponse(json.dumps(statuses), content_type='application/json') + return HttpResponse(json.dumps(statuses), content_type="application/json") def export_download(request, username, id_string, export_type, filename): @@ -561,31 +605,38 @@ def export_download(request, username, id_string, export_type, filename): Export download view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) # find the export entry in the db export = get_object_or_404(Export, xform=xform, filename=filename) - if (export_type == Export.GOOGLE_SHEETS_EXPORT or - export_type == Export.EXTERNAL_EXPORT) and \ - export.export_url is not None: + if ( + export_type == Export.GOOGLE_SHEETS_EXPORT + or export_type == Export.EXTERNAL_EXPORT + ) and export.export_url is not None: return HttpResponseRedirect(export.export_url) ext, mime_type = export_def_from_filename(export.filename) audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export '%(filename)s' " - "on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded %(export_type)s export '%(filename)s' " "on '%(id_string)s'.") + % { + "export_type": export.export_type.upper(), + "filename": export.filename, + "id_string": xform.id_string, + }, + audit, + request, + ) + if request.GET.get("raw"): id_string = None default_storage = get_storage_class()() @@ -597,7 +648,8 @@ def export_download(request, username, id_string, export_type, filename): name=basename, extension=ext, file_path=export.filepath, - show_date=False) + show_date=False, + ) return response @@ -608,33 +660,41 @@ def delete_export(request, username, id_string, export_type): Delete export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - export_id = request.POST.get('export_id') + export_id = request.POST.get("export_id") # find the export entry in the db export = get_object_or_404(Export, id=export_id) export.delete() audit = {"xform": xform.id_string, "export_type": export.export_type} - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Deleted %(export_type)s export '%(filename)s'" - " on '%(id_string)s'.") % { - 'export_type': export.export_type.upper(), - 'filename': export.filename, - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Deleted %(export_type)s export '%(filename)s'" " on '%(id_string)s'.") + % { + "export_type": export.export_type.upper(), + "filename": export.filename, + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect( reverse( export_list, kwargs={ "username": username, "id_string": id_string, - "export_type": export_type - })) + "export_type": export_type, + }, + ) + ) def zip_export(request, username, id_string): @@ -642,12 +702,12 @@ def zip_export(request, username, id_string): Zip export view. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - if request.GET.get('raw'): + return HttpResponseForbidden(_("Not shared.")) + if request.GET.get("raw"): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) @@ -656,21 +716,35 @@ def zip_export(request, username, id_string): try: zip_file = create_attachments_zipfile(attachments) audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created ZIP export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded ZIP export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) - if request.GET.get('raw'): + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded ZIP export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + if request.GET.get("raw"): id_string = None - response = response_with_mimetype_and_name('zip', id_string) + response = response_with_mimetype_and_name("zip", id_string) response.write(FileWrapper(zip_file)) - response['Content-Length'] = zip_file.tell() + response["Content-Length"] = zip_file.tell() zip_file.seek(0) finally: if zip_file: @@ -685,29 +759,42 @@ def kml_export(request, username, id_string): """ # read the locations from the database owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) helper_auth_helper(request) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) - data = {'data': kml_export_data(id_string, user=owner, xform=xform)} + return HttpResponseForbidden(_("Not shared.")) + data = {"data": kml_export_data(id_string, user=owner, xform=xform)} response = render( - request, - "survey.kml", - data, - content_type="application/vnd.google-earth.kml+xml") - response['Content-Disposition'] = \ - generate_content_disposition_header(id_string, 'kml') + request, "survey.kml", data, content_type="application/vnd.google-earth.kml+xml" + ) + response["Content-Disposition"] = generate_content_disposition_header( + id_string, "kml" + ) audit = {"xform": xform.id_string, "export_type": Export.KML_EXPORT} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created KML export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) # log download as well - audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded KML export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_DOWNLOADED, + request.user, + owner, + _("Downloaded KML export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return response @@ -725,24 +812,22 @@ def google_xls_export(request, username, id_string): pass else: token = token_storage.token - elif request.session.get('access_token'): - token = request.session.get('access_token') + elif request.session.get("access_token"): + token = request.session.get("access_token") if token is None: request.session["google_redirect_url"] = reverse( - google_xls_export, - kwargs={'username': username, - 'id_string': id_string}) + google_xls_export, kwargs={"username": username, "id_string": id_string} + ) return HttpResponseRedirect(google_flow.step1_get_authorize_url()) owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'user': owner, 'id_string__iexact': id_string}) + xform = get_form({"user": owner, "id_string__iexact": id_string}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - is_valid, data_dictionary = set_instances_for_export( - id_string, owner, request) + is_valid, data_dictionary = set_instances_for_export(id_string, owner, request) if not is_valid: return data_dictionary @@ -755,10 +840,17 @@ def google_xls_export(request, username, id_string): url = None os.unlink(tmp.name) audit = {"xform": xform.id_string, "export_type": "google"} - audit_log(Actions.EXPORT_CREATED, request.user, owner, - _("Created Google Docs export on '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.EXPORT_CREATED, + request.user, + owner, + _("Created Google Docs export on '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return HttpResponseRedirect(url) @@ -768,51 +860,59 @@ def data_view(request, username, id_string): Data view displays submission data. """ owner = get_object_or_404(User, username__iexact=username) - xform = get_form({'id_string__iexact': id_string, 'user': owner}) + xform = get_form({"id_string__iexact": id_string, "user": owner}) if not has_permission(xform, owner, request): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - data = {'owner': owner, 'xform': xform} + data = {"owner": owner, "xform": xform} audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, owner, - _("Requested data view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + owner, + _("Requested data view for '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) return render(request, "data_view.html", data) -def attachment_url(request, size='medium'): +def attachment_url(request, size="medium"): """ Redirects to image attachment of the specified size, defaults to 'medium'. """ - media_file = request.GET.get('media_file') - no_redirect = request.GET.get('no_redirect') + media_file = request.GET.get("media_file") + no_redirect = request.GET.get("no_redirect") if not media_file: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) result = Attachment.objects.filter(media_file=media_file).order_by()[0:1] if not result: - return HttpResponseNotFound(_(u'Attachment not found')) + return HttpResponseNotFound(_("Attachment not found")) attachment = result[0] - if size == 'original' and no_redirect == 'true': + if size == "original" and no_redirect == "true": response = response_with_mimetype_and_name( attachment.mimetype, attachment.name, extension=attachment.extension, - file_path=attachment.media_file.name) + file_path=attachment.media_file.name, + ) return response - if not attachment.mimetype.startswith('image'): + if not attachment.mimetype.startswith("image"): return redirect(attachment.media_file.url) media_url = image_url(attachment, size) if media_url: return redirect(media_url) - return HttpResponseNotFound(_(u'Error: Attachment not found')) + return HttpResponseNotFound(_("Error: Attachment not found")) def instance(request, username, id_string): @@ -821,26 +921,41 @@ def instance(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) audit = { "xform": xform.id_string, } - audit_log(Actions.FORM_DATA_VIEWED, request.user, xform.user, - _("Requested instance view for '%(id_string)s'.") % { - 'id_string': xform.id_string, - }, audit, request) + audit_log( + Actions.FORM_DATA_VIEWED, + request.user, + xform.user, + _("Requested instance view for '%(id_string)s'.") + % { + "id_string": xform.id_string, + }, + audit, + request, + ) - return render(request, 'instance.html', { - 'username': username, - 'id_string': id_string, - 'xform': xform, - 'can_edit': can_edit - }) + return render( + request, + "instance.html", + { + "username": username, + "id_string": id_string, + "xform": xform, + "can_edit": can_edit, + }, + ) def charts(request, username, id_string): @@ -849,20 +964,24 @@ def charts(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) try: - lang_index = int(request.GET.get('lang', 0)) + lang_index = int(request.GET.get("lang", 0)) except ValueError: lang_index = 0 try: - page = int(request.GET.get('page', 0)) + page = int(request.GET.get("page", 0)) except ValueError: page = 0 else: @@ -871,14 +990,13 @@ def charts(request, username, id_string): summaries = build_chart_data(xform, lang_index, page) if request.is_ajax(): - template = 'charts_snippet.html' + template = "charts_snippet.html" else: - template = 'charts.html' + template = "charts.html" - return render(request, template, - {'xform': xform, - 'summaries': summaries, - 'page': page + 1}) + return render( + request, template, {"xform": xform, "summaries": summaries, "page": page + 1} + ) def stats_tables(request, username, id_string): @@ -887,10 +1005,14 @@ def stats_tables(request, username, id_string): """ # pylint: disable=W0612 xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + username, id_string, request + ) # no access - if not (xform.shared_data or can_view - or request.session.get('public_link') == xform.uuid): - return HttpResponseForbidden(_(u'Not shared.')) - - return render(request, 'stats_tables.html', {'xform': xform}) + if not ( + xform.shared_data + or can_view + or request.session.get("public_link") == xform.uuid + ): + return HttpResponseForbidden(_("Not shared.")) + + return render(request, "stats_tables.html", {"xform": xform}) From a8b83b95993dfca9d3b4a6d134f6ec840e05128b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Apr 2022 05:39:47 +0300 Subject: [PATCH 015/234] Update requirements/base.pip --- requirements/base.pip | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index 4604b22f8c..022d9daa7f 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -4,7 +4,6 @@ # # pip-compile --output-file=requirements/base.pip requirements/base.in # --e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform -e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router @@ -19,6 +18,11 @@ # via -r requirements/base.in -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient # via -r requirements/base.in +-e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform + # via + # -r requirements/base.in + # onadata + # pyfloip alabaster==0.7.12 # via sphinx amqp==5.1.1 @@ -35,15 +39,15 @@ attrs==21.4.0 # via # jsonlines # jsonschema -babel==2.9.1 +babel==2.10.1 # via sphinx backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery -boto3==1.21.43 +boto3==1.21.46 # via tabulator -botocore==1.24.43 +botocore==1.24.46 # via # boto3 # s3transfer @@ -84,9 +88,7 @@ cryptography==36.0.2 datapackage==1.15.2 # via pyfloip defusedxml==0.7.1 - # via - # djangorestframework-xml - # pyxform + # via djangorestframework-xml deprecated==1.2.13 # via # jwcrypto @@ -242,7 +244,6 @@ oauthlib==3.2.0 openpyxl==3.0.9 # via # onadata - # pyxform # tabulator packaging==21.3 # via @@ -271,7 +272,7 @@ pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 -pygments==2.11.2 +pygments==2.12.0 # via sphinx pyjwt[crypto]==2.3.0 # via @@ -405,7 +406,6 @@ wrapt==1.14.0 xlrd==2.0.1 # via # onadata - # pyxform # tabulator xlwt==1.3.0 # via onadata From d04ebe9968ad2766cfbed3d1b155db7b61bd3364 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Apr 2022 07:11:48 +0300 Subject: [PATCH 016/234] Remove xlrd package in data_dictionary.py --- onadata/apps/viewer/models/data_dictionary.py | 3 +-- requirements/base.pip | 4 +--- setup.cfg | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index b2a92961c2..1a1bb1042d 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -6,7 +6,6 @@ from io import BytesIO, StringIO import unicodecsv as csv -import xlrd import openpyxl from builtins import str as text from django.core.files.storage import get_storage_class @@ -120,7 +119,7 @@ def sheet_to_csv(xls_content, sheet_name): except ValueError: pass elif sheet.cell(row, index).is_date: - val = xlrd.xldate_as_datetime(val, workbook.datemode).isoformat() + val = val.strftime("%Y-%m-%d").isoformat() row_values.append(val) writer.writerow([v for v, m in zip(row_values, mask) if m]) else: diff --git a/requirements/base.pip b/requirements/base.pip index 022d9daa7f..1c6f62a204 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -404,9 +404,7 @@ wcwidth==0.2.5 wrapt==1.14.0 # via deprecated xlrd==2.0.1 - # via - # onadata - # tabulator + # via tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 diff --git a/setup.cfg b/setup.cfg index 398ac0baa8..bb0bae1485 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,7 +77,6 @@ install_requires = Markdown #others unicodecsv - xlrd xlwt openpyxl dpath From 0aa473bae346474bd956557936ee31ed20853f59 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 08:56:45 +0300 Subject: [PATCH 017/234] Remove deprecated django.db.backends.postgresql_psycopg2 Signed-off-by: Kipchirchir Sigei --- onadata/libs/data/query.py | 2 +- onadata/settings/production_example.py | 2 +- onadata/settings/staging_example.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/onadata/libs/data/query.py b/onadata/libs/data/query.py index 372238f04a..c58a19a4e6 100644 --- a/onadata/libs/data/query.py +++ b/onadata/libs/data/query.py @@ -263,4 +263,4 @@ def is_date_field(xform, field): @property def using_postgres(): return settings.DATABASES[ - 'default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2' + 'default']['ENGINE'] == 'django.db.backends.postgresql' diff --git a/onadata/settings/production_example.py b/onadata/settings/production_example.py index 0aa7eb1bd4..5023d94d7f 100644 --- a/onadata/settings/production_example.py +++ b/onadata/settings/production_example.py @@ -18,7 +18,7 @@ # your actual production settings go here...,. DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': 'formhub', 'USER': 'formhub_prod', # the password must be stored in an environment variable diff --git a/onadata/settings/staging_example.py b/onadata/settings/staging_example.py index fe7b2f1f27..630a9fab03 100644 --- a/onadata/settings/staging_example.py +++ b/onadata/settings/staging_example.py @@ -10,7 +10,7 @@ # postgres DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': 'formhub_dev', 'USER': 'formhub_dev', 'PASSWORD': '12345678', From cf42e8ef6c43b3cb6267e2c012e4e16f2c69b296 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 09:00:31 +0300 Subject: [PATCH 018/234] Fix dataview query_iterator bug Signed-off-by: Kipchirchir Sigei --- onadata/apps/api/tests/viewsets/test_abstract_viewset.py | 2 -- onadata/apps/logger/models/data_view.py | 3 ++- onadata/apps/logger/models/xform.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 8d028fd7dd..f5673a0f2a 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -465,9 +465,7 @@ def _create_dataview(self, data=None, project=None, xform=None): "query": '[{"column":"age","filter":">","value":"20"},' '{"column":"age","filter":"<","value":"50"}]', } - request = self.factory.post("/", data=data, **self.extra) - response = view(request) self.assertEquals(response.status_code, 201) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index af7e9c1c70..58041e6566 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -3,6 +3,7 @@ DataView model class """ import datetime +import json from builtins import str as text from django.conf import settings @@ -292,7 +293,7 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): yield dict(zip(fields, row)) else: for row in cursor.fetchall(): - yield dict(zip(fields, [row[0].get(f) for f in fields])) + yield dict(zip(fields, [json.loads(row[0]).get(f) for f in fields])) @classmethod def generate_query_string( diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 6f3503fa64..c297771834 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -70,7 +70,7 @@ title_pattern = re.compile(r"(.*?)") -cmp = lambda x, y: (x > y) - (x < y) +def cmp(x, y): (x > y) - (x < y) def question_types_to_exclude(_type): @@ -795,7 +795,7 @@ class XForm(XFormMixin, BaseModel): ) metadata_set = GenericRelation( "main.MetaData", - content_type_field="content_type_id", + content_type_field="content_type", object_id_field="object_id", ) has_hxl_support = models.BooleanField(default=False) From cef1f01df1957fa6ae3f9db295716562c507e6a8 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Apr 2022 09:02:44 +0300 Subject: [PATCH 019/234] Upgrade pyxform to version 1.10.0 --- requirements/base.in | 1 - requirements/base.pip | 17 ++++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 703d532072..81444c0dc4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,7 +2,6 @@ -e . # installed from Git --e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform -e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest -e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router diff --git a/requirements/base.pip b/requirements/base.pip index 4a0aacd876..2c4756f624 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -18,11 +18,6 @@ # via -r requirements/base.in -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient # via -r requirements/base.in --e git+https://github.com/XLSForm/pyxform.git@f4ce2ec7f90d3e197b9b5b58fecccabe31d213f8#egg=pyxform - # via - # -r requirements/base.in - # onadata - # pyfloip alabaster==0.7.12 # via sphinx amqp==5.1.1 @@ -88,7 +83,9 @@ cryptography==36.0.2 datapackage==1.15.2 # via pyfloip defusedxml==0.7.1 - # via djangorestframework-xml + # via + # djangorestframework-xml + # pyxform deprecated==1.2.13 # via # jwcrypto @@ -307,6 +304,10 @@ pytz==2022.1 # djangorestframework # fleming # onadata +pyxform==1.10.0 + # via + # onadata + # pyfloip raven==6.10.0 # via onadata recaptcha-client==1.0.6 @@ -405,7 +406,9 @@ wcwidth==0.2.5 wrapt==1.14.0 # via deprecated xlrd==2.0.1 - # via tabulator + # via + # pyxform + # tabulator xlwt==1.3.0 # via onadata xmltodict==0.12.0 From 02dd9c9d8b5ccb6aeef19ec8db0774e84fd89074 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 09:43:01 +0300 Subject: [PATCH 020/234] Remove deprecated django.contrib.postgres.fields.JSONField Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/data_view.py | 2 +- onadata/apps/logger/models/instance.py | 3 +-- onadata/apps/logger/models/osmdata.py | 2 +- onadata/apps/logger/models/project.py | 3 +-- onadata/apps/logger/models/widget.py | 2 +- onadata/apps/main/models/audit.py | 2 +- onadata/apps/main/models/user_profile.py | 2 +- onadata/apps/viewer/models/export.py | 2 +- 8 files changed, 8 insertions(+), 10 deletions(-) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 58041e6566..96db6c8d3e 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -8,8 +8,8 @@ from django.conf import settings from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField from django.db import connection +from django.db.models import JSONField from django.db.models.signals import post_delete, post_save from django.utils import timezone from django.utils.translation import ugettext as _ diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 721fa7ff29..c5f37fa6ef 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -11,10 +11,9 @@ from django.contrib.auth.models import User from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point -from django.contrib.postgres.fields import JSONField from django.core.cache import cache from django.db import connection, transaction -from django.db.models import Q +from django.db.models import Q, JSONField from django.db.models.signals import post_delete, post_save from django.urls import reverse from django.utils import timezone diff --git a/onadata/apps/logger/models/osmdata.py b/onadata/apps/logger/models/osmdata.py index 9cc0ec7ce9..352bc496bf 100644 --- a/onadata/apps/logger/models/osmdata.py +++ b/onadata/apps/logger/models/osmdata.py @@ -2,8 +2,8 @@ """ OSM Data model class """ +from django.db.models import JSONField from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField class OsmData(models.Model): diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index 81c03b1bd1..a87ba9cb90 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -4,10 +4,9 @@ """ from django.conf import settings from django.contrib.auth.models import User -from django.contrib.postgres.fields import JSONField from django.core.exceptions import ValidationError from django.db import models, transaction -from django.db.models import Prefetch +from django.db.models import Prefetch, JSONField from django.db.models.signals import post_save from django.utils import timezone from six import python_2_unicode_compatible diff --git a/onadata/apps/logger/models/widget.py b/onadata/apps/logger/models/widget.py index 67b447ecc1..44e878afac 100644 --- a/onadata/apps/logger/models/widget.py +++ b/onadata/apps/logger/models/widget.py @@ -1,9 +1,9 @@ from builtins import str as text +from django.db.models import JSONField from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.contrib.postgres.fields import JSONField from ordered_model.models import OrderedModel from querybuilder.fields import AvgField, CountField, SimpleField, SumField from querybuilder.query import Query diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 3a4d0ef4d2..c6bf06e300 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -3,7 +3,7 @@ from django.db import models from django.db import connection -from django.contrib.postgres.fields import JSONField +from django.db.models import JSONField from django.utils.translation import ugettext as _ DEFAULT_LIMIT = 1000 diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 117d118b0f..013a1d13bc 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -5,8 +5,8 @@ import requests from django.conf import settings from django.contrib.auth.models import User -from django.contrib.postgres.fields import JSONField from django.db import models +from django.db.models import JSONField from django.db.models.signals import post_save, pre_save from django.utils.translation import ugettext_lazy from guardian.shortcuts import get_perms_for_model, assign_perm diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 815d58b34c..3e179f1a48 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -8,8 +8,8 @@ from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class -from django.contrib.postgres.fields import JSONField from django.db import models +from django.db.models import JSONField from django.db.models.signals import post_delete from django.utils.translation import ugettext as _ From deefb2d7b3e5335601ee55f36758cac0be23ab81 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 09:52:46 +0300 Subject: [PATCH 021/234] Remove deprecated python_2_unicode_compatible decorator Signed-off-by: Kipchirchir Sigei --- onadata/apps/api/models/organization_profile.py | 2 -- onadata/apps/api/models/team.py | 3 --- onadata/apps/api/models/temp_token.py | 2 -- onadata/apps/logger/models/data_view.py | 2 -- onadata/apps/logger/models/instance.py | 3 --- onadata/apps/logger/models/open_data.py | 2 -- onadata/apps/logger/models/project.py | 2 -- onadata/apps/logger/models/survey_type.py | 2 -- onadata/apps/logger/models/xform.py | 3 +-- onadata/apps/logger/xform_instance_parser.py | 5 ----- onadata/apps/main/models/user_profile.py | 2 -- onadata/apps/restservice/models.py | 3 --- onadata/apps/viewer/models/data_dictionary.py | 2 -- onadata/apps/viewer/models/export.py | 4 ---- 14 files changed, 1 insertion(+), 36 deletions(-) diff --git a/onadata/apps/api/models/organization_profile.py b/onadata/apps/api/models/organization_profile.py index 50179b9208..d0dd1a3dad 100644 --- a/onadata/apps/api/models/organization_profile.py +++ b/onadata/apps/api/models/organization_profile.py @@ -6,7 +6,6 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models.signals import post_delete, post_save -from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -73,7 +72,6 @@ def _post_save_create_owner_team(sender, instance, created, **kwargs): create_owner_team_and_assign_permissions(instance) -@python_2_unicode_compatible class OrganizationProfile(UserProfile): """Organization: Extends the user profile for organization specific info diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index 4cb3f99c51..a96bde6f3f 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -1,5 +1,3 @@ -from six import python_2_unicode_compatible - from django.db import models from django.db.models.signals import post_save from django.contrib.auth.models import User, Group @@ -8,7 +6,6 @@ from onadata.apps.logger.models.project import Project -@python_2_unicode_compatible class Team(Group): """ TODO: documentation diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index 3f46fc53ae..ef782ad901 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -7,12 +7,10 @@ from django.conf import settings from django.db import models -from six import python_2_unicode_compatible AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") -@python_2_unicode_compatible class TempToken(models.Model): """ diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 96db6c8d3e..a0983dd3dc 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -13,7 +13,6 @@ from django.db.models.signals import post_delete, post_save from django.utils import timezone from django.utils.translation import ugettext as _ -from six import python_2_unicode_compatible from onadata.apps.viewer.parsed_instance_tools import get_where_clause from onadata.libs.models.sorting import ( @@ -98,7 +97,6 @@ def has_attachments_fields(data_view): return False -@python_2_unicode_compatible class DataView(models.Model): """ Model to provide filtered access to the underlying data of an XForm diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index c5f37fa6ef..609aa51480 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -18,7 +18,6 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import ugettext as _ -from six import python_2_unicode_compatible from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview @@ -115,7 +114,6 @@ def _get_tag_or_element_type_xpath(xform, tag): return elems[0].get_abbreviated_xpath() if elems else tag -@python_2_unicode_compatible class FormInactiveError(Exception): """Exception class for inactive forms""" @@ -123,7 +121,6 @@ def __str__(self): return _("Form is inactive") -@python_2_unicode_compatible class FormIsMergedDatasetError(Exception): """Exception class for merged datasets""" diff --git a/onadata/apps/logger/models/open_data.py b/onadata/apps/logger/models/open_data.py index c7c85f852d..8c57f717c1 100644 --- a/onadata/apps/logger/models/open_data.py +++ b/onadata/apps/logger/models/open_data.py @@ -7,12 +7,10 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from six import python_2_unicode_compatible from onadata.libs.utils.common_tools import get_uuid -@python_2_unicode_compatible class OpenData(models.Model): """ OpenData model represents a way to access private datasets without diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index a87ba9cb90..c0ff477d57 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -9,7 +9,6 @@ from django.db.models import Prefetch, JSONField from django.db.models.signals import post_save from django.utils import timezone -from six import python_2_unicode_compatible from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model @@ -78,7 +77,6 @@ def get_queryset(self): ) -@python_2_unicode_compatible class Project(BaseModel): """ Project model class diff --git a/onadata/apps/logger/models/survey_type.py b/onadata/apps/logger/models/survey_type.py index d766b35b96..5a25fe0547 100644 --- a/onadata/apps/logger/models/survey_type.py +++ b/onadata/apps/logger/models/survey_type.py @@ -3,10 +3,8 @@ Survey type model class """ from django.db import models -from six import python_2_unicode_compatible -@python_2_unicode_compatible class SurveyType(models.Model): """ Survey type model class diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index c297771834..d7b88d0377 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -19,7 +19,7 @@ from django.utils import timezone from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from six import iteritems, itervalues, python_2_unicode_compatible +from six import iteritems, itervalues from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question @@ -733,7 +733,6 @@ def get_osm_survey_xpaths(self): ] -@python_2_unicode_compatible class XForm(XFormMixin, BaseModel): CLONED_SUFFIX = "_cloned" MAX_ID_LENGTH = 100 diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index d6ceace3ce..7d23840bbf 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -2,7 +2,6 @@ import re import dateutil.parser from builtins import str as text -from six import python_2_unicode_compatible from xml.dom import minidom, Node from django.utils.encoding import smart_text, smart_str @@ -15,25 +14,21 @@ class XLSFormError(Exception): pass -@python_2_unicode_compatible class DuplicateInstance(Exception): def __str__(self): return _("Duplicate Instance") -@python_2_unicode_compatible class InstanceInvalidUserError(Exception): def __str__(self): return _("Could not determine the user.") -@python_2_unicode_compatible class InstanceParseError(Exception): def __str__(self): return _("The instance could not be parsed.") -@python_2_unicode_compatible class InstanceEmptyError(InstanceParseError): def __str__(self): return _("Empty instance") diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 013a1d13bc..0862552ad9 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -13,7 +13,6 @@ from guardian.models import UserObjectPermissionBase from guardian.models import GroupObjectPermissionBase from rest_framework.authtoken.models import Token -from six import python_2_unicode_compatible from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists from onadata.apps.main.signals import ( @@ -25,7 +24,6 @@ REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" -@python_2_unicode_compatible class UserProfile(models.Model): """ Userprofile model diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 24101e8980..2632a70afc 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -4,8 +4,6 @@ """ import importlib -from six import python_2_unicode_compatible - from django.conf import settings from django.db import models from django.db.models.signals import post_delete, post_save @@ -17,7 +15,6 @@ from onadata.libs.utils.common_tags import GOOGLE_SHEET, TEXTIT -@python_2_unicode_compatible class RestService(models.Model): """ Properties for an external service. diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 1a1bb1042d..dae0466c76 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -18,7 +18,6 @@ from pyxform.builder import create_survey_element_from_dict from pyxform.utils import has_external_choices from pyxform.xls2json import parse_file_to_json -from six import python_2_unicode_compatible from onadata.apps.logger.models.xform import XForm, check_version_set, check_xform_uuid from onadata.apps.logger.xform_instance_parser import XLSFormError @@ -137,7 +136,6 @@ def upload_to(instance, filename, username=None): return os.path.join(username, "xls", os.path.split(filename)[1]) -@python_2_unicode_compatible class DataDictionary(XForm): # pylint: disable=too-many-instance-attributes """ DataDictionary model class. diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 3e179f1a48..967bb75d77 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -4,7 +4,6 @@ """ import os -from six import python_2_unicode_compatible from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class @@ -45,7 +44,6 @@ def get_export_options_query_kwargs(options): return options_kwargs -@python_2_unicode_compatible class ExportTypeError(Exception): """ ExportTypeError exception class. @@ -55,7 +53,6 @@ def __str__(self): return _("Invalid export type specified") -@python_2_unicode_compatible class ExportConnectionError(Exception): """ ExportConnectionError exception class. @@ -65,7 +62,6 @@ def __str__(self): return _("Export server is down.") -@python_2_unicode_compatible class Export(models.Model): """ Class representing a data export from an XForm From aa5969f309c0e98af035a34aa96284f2cc0a03e9 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:11:05 +0300 Subject: [PATCH 022/234] Squash logger application migrations --- .../migrations/0001_pre-django-3-upgrade.py | 539 ++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py diff --git a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py new file mode 100644 index 0000000000..8c18800737 --- /dev/null +++ b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py @@ -0,0 +1,539 @@ +# Generated by Django 3.2.13 on 2022-04-25 06:54 + +import datetime +from django.conf import settings +import django.contrib.gis.db.models.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +from django.utils.timezone import utc +import onadata.apps.logger.models.attachment +import onadata.apps.logger.models.xform +import onadata.libs.utils.common_tools +from onadata.apps.logger.models import Instance +from onadata.libs.utils.model_tools import queryset_iterator +from onadata.libs.utils.logger_tools import create_xform_version +import taggit.managers +from hashlib import md5 + + +def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 + """ + Recalculate all XForm hashes. + """ + XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, + deleted_at__isnull=True).only('xml') + count = xforms.count() + counter = 0 + + for xform in queryset_iterator(xforms, 500): + hash_value = md5(xform.xml.encode('utf8')).hexdigest() + xform.hash = f"md5:{hash_value}" + xform.save(update_fields=['hash']) + counter += 1 + if counter % 500 == 0: + print(f"Processed {counter} of {count} forms.") + + print(f"Processed {counter} forms.") + + +def generate_uuid_if_missing(apps, schema_editor): + """ + Generate uuids for XForms without them + """ + XForm = apps.get_model('logger', 'XForm') + + for xform in XForm.objects.filter(uuid=''): + xform.uuid = onadata.libs.utils.common_tools.get_uuid() + xform.save() + + +def regenerate_instance_json(apps, schema_editor): + """ + Regenerate Instance JSON + """ + for inst in Instance.objects.filter( + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True): + inst.json = inst.get_full_dict(load_existing=False) + inst.save() + + +def create_initial_xform_version(apps, schema_editor): + """ + Creates an XFormVersion object for an XForm that has no + Version + """ + queryset = onadata.apps.logger.models.xform.XForm.objects.filter( + downloadable=True, + deleted_at__isnull=True + ) + for xform in queryset.iterator(): + if xform.version: + create_xform_version(xform, xform.user) + + +class Migration(migrations.Migration): + + replaces = [('logger', '0001_initial'), ('logger', '0002_auto_20150717_0048'), ('logger', '0003_dataview_instances_with_geopoints'), ('logger', '0004_auto_20150910_0056'), ('logger', '0005_auto_20151015_0758'), ('logger', '0006_auto_20151106_0130'), ('logger', '0007_osmdata_field_name'), ('logger', '0008_osmdata_osm_type'), ('logger', '0009_auto_20151111_0438'), ('logger', '0010_attachment_file_size'), ('logger', '0011_dataview_matches_parent'), ('logger', '0012_auto_20160114_0708'), ('logger', '0013_note_created_by'), ('logger', '0014_note_instance_field'), ('logger', '0015_auto_20160222_0559'), ('logger', '0016_widget_aggregation'), ('logger', '0017_auto_20160224_0130'), ('logger', '0018_auto_20160301_0330'), ('logger', '0019_auto_20160307_0256'), ('logger', '0020_auto_20160408_0325'), ('logger', '0021_auto_20160408_0919'), ('logger', '0022_auto_20160418_0518'), ('logger', '0023_auto_20160419_0403'), ('logger', '0024_xform_has_hxl_support'), ('logger', '0025_xform_last_updated_at'), ('logger', '0026_auto_20160913_0239'), ('logger', '0027_auto_20161201_0730'), ('logger', '0028_auto_20170221_0838'), ('logger', '0029_auto_20170221_0908'), ('logger', '0030_auto_20170227_0137'), ('logger', '0028_auto_20170217_0502'), ('logger', '0031_merge'), ('logger', '0032_project_deleted_at'), ('logger', '0033_auto_20170705_0159'), ('logger', '0034_mergedxform'), ('logger', '0035_auto_20170712_0529'), ('logger', '0036_xform_is_merged_dataset'), ('logger', '0034_auto_20170814_0432'), ('logger', '0037_merge_20170825_0238'), ('logger', '0038_auto_20170828_1718'), ('logger', '0039_auto_20170909_2052'), ('logger', '0040_auto_20170912_1504'), ('logger', '0041_auto_20170912_1512'), ('logger', '0042_xform_hash'), ('logger', '0043_auto_20171010_0403'), ('logger', '0044_xform_hash_sql_update'), ('logger', '0045_attachment_name'), ('logger', '0046_auto_20180314_1618'), ('logger', '0047_dataview_deleted_at'), ('logger', '0048_dataview_deleted_by'), ('logger', '0049_xform_deleted_by'), ('logger', '0050_project_deleted_by'), ('logger', '0051_auto_20180522_1118'), ('logger', '0052_auto_20180805_2233'), ('logger', '0053_submissionreview'), ('logger', '0054_instance_has_a_review'), ('logger', '0055_auto_20180904_0713'), ('logger', '0056_auto_20190125_0517'), ('logger', '0057_xform_public_key'), ('logger', '0058_auto_20191211_0900'), ('logger', '0059_attachment_deleted_by'), ('logger', '0060_auto_20200305_0357'), ('logger', '0061_auto_20200713_0814'), ('logger', '0062_auto_20210202_0248'), ('logger', '0063_xformversion'), ('logger', '0064_auto_20210304_0314')] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('auth', '0001_initial'), + ('taggit', '0001_initial'), + ('contenttypes', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('shared', models.BooleanField(default=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_owner', to=settings.AUTH_USER_MODEL)), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_org', to=settings.AUTH_USER_MODEL)), + ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), + ('user_stars', models.ManyToManyField(related_name='project_stars', to=settings.AUTH_USER_MODEL)), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'permissions': (('view_project', 'Can view project'), ('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data')), + 'unique_together': {('name', 'organization')}, + }, + ), + migrations.CreateModel( + name='SurveyType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('slug', models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name='XForm', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xls', models.FileField(null=True, upload_to=onadata.apps.logger.models.xform.upload_to)), + ('json', models.TextField(default='')), + ('description', models.TextField(blank=True, default='', null=True)), + ('xml', models.TextField()), + ('require_auth', models.BooleanField(default=False)), + ('shared', models.BooleanField(default=False)), + ('shared_data', models.BooleanField(default=False)), + ('downloadable', models.BooleanField(default=True)), + ('allows_sms', models.BooleanField(default=False)), + ('encrypted', models.BooleanField(default=False)), + ('sms_id_string', models.SlugField(default=b'', editable=False, max_length=100, verbose_name='SMS ID')), + ('id_string', models.SlugField(editable=False, max_length=100, verbose_name='ID')), + ('title', models.CharField(editable=False, max_length=255)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ('last_submission_time', models.DateTimeField(blank=True, null=True)), + ('has_start_time', models.BooleanField(default=False)), + ('uuid', models.CharField(default='', max_length=32)), + ('bamboo_dataset', models.CharField(default='', max_length=60)), + ('instances_with_geopoints', models.BooleanField(default=False)), + ('instances_with_osm', models.BooleanField(default=False)), + ('num_of_submissions', models.IntegerField(default=0)), + ('version', models.CharField(blank=True, max_length=255, null=True)), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), + ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), + ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='xforms', to=settings.AUTH_USER_MODEL)), + ('has_hxl_support', models.BooleanField(default=False)), + ('last_updated_at', models.DateTimeField(auto_now=True, default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc))), + ('is_merged_dataset', models.BooleanField(default=False)), + ('hash', models.CharField(blank=True, default=None, max_length=36, null=True, verbose_name='Hash')), + ], + options={ + 'ordering': ('pk',), + 'verbose_name': 'XForm', + 'verbose_name_plural': 'XForms', + 'permissions': (('view_xform', 'Can view associated data'), ('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), + 'unique_together': {('user', 'id_string', 'project'), ('user', 'sms_id_string', 'project')}, + }, + ), + migrations.CreateModel( + name='Instance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('json', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('xml', models.TextField()), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('status', models.CharField(default='submitted_via_web', max_length=20)), + ('uuid', models.CharField(db_index=True, default='', max_length=249)), + ('version', models.CharField(max_length=255, null=True)), + ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(null=True, srid=4326)), + ('survey_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.surveytype')), + ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), + ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instances', to=settings.AUTH_USER_MODEL)), + ('xform', models.ForeignKey(default=328, on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform')), + ('last_edited', models.DateTimeField(default=None, null=True)), + ('media_all_received', models.NullBooleanField(default=True, verbose_name='Received All Media Attachemts')), + ('media_count', models.PositiveIntegerField(default=0, null=True, verbose_name='Received Media Attachments')), + ('total_media', models.PositiveIntegerField(default=0, null=True, verbose_name='Total Media Attachments')), + ('checksum', models.CharField(blank=True, db_index=True, max_length=64, null=True)), + ], + options={ + 'unique_together': {('xform', 'uuid')}, + }, + ), + migrations.CreateModel( + name='ProjectUserObjectPermission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + 'unique_together': {('user', 'permission', 'content_object')}, + }, + ), + migrations.CreateModel( + name='ProjectGroupObjectPermission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ], + options={ + 'abstract': False, + 'unique_together': {('group', 'permission', 'content_object')}, + }, + ), + migrations.CreateModel( + name='XFormUserObjectPermission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + 'unique_together': {('user', 'permission', 'content_object')}, + }, + ), + migrations.CreateModel( + name='XFormGroupObjectPermission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ], + options={ + 'abstract': False, + 'unique_together': {('group', 'permission', 'content_object')}, + }, + ), + migrations.CreateModel( + name='Note', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('note', models.TextField()), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notes', to='logger.instance')), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('instance_field', models.TextField(blank=True, null=True)), + ], + options={ + 'permissions': (('view_note', 'View note'),), + }, + ), + migrations.CreateModel( + name='DataView', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('columns', django.contrib.postgres.fields.jsonb.JSONField()), + ('query', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), + ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), + ('instances_with_geopoints', models.BooleanField(default=False)), + ('matches_parent', models.BooleanField(default=False)), + ], + options={ + 'verbose_name': 'Data View', + 'verbose_name_plural': 'Data Views', + }, + ), + migrations.CreateModel( + name='OsmData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xml', models.TextField()), + ('osm_id', models.CharField(max_length=20)), + ('tags', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(srid=4326)), + ('filename', models.CharField(max_length=255)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='osm_data', to='logger.instance')), + ('field_name', models.CharField(blank=True, default=b'', max_length=255)), + ('osm_type', models.CharField(default=b'way', max_length=10)), + ], + options={ + 'unique_together': {('instance', 'field_name')}, + }, + ), + migrations.CreateModel( + name='Widget', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField()), + ('widget_type', models.CharField(choices=[(b'charts', b'Charts')], default=b'charts', max_length=25)), + ('view_type', models.CharField(max_length=50)), + ('column', models.CharField(max_length=255)), + ('group_by', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('title', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('description', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('key', models.CharField(db_index=True, max_length=32, unique=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ('order', models.PositiveIntegerField(db_index=True, default=0, editable=False)), + ('aggregation', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('metadata', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict)), + ], + options={ + 'ordering': ('order',), + }, + ), + migrations.CreateModel( + name='OpenData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('uuid', models.CharField(default=onadata.libs.utils.common_tools.get_uuid, max_length=32, unique=True)), + ('object_id', models.PositiveIntegerField(blank=True, null=True)), + ('active', models.BooleanField(default=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ], + ), + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('media_file', models.FileField(max_length=255, upload_to=onadata.apps.logger.models.attachment.upload_to)), + ('mimetype', models.CharField(blank=True, default=b'', max_length=100)), + ('extension', models.CharField(db_index=True, default='non', max_length=10)), + ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='logger.instance')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True)), + ('date_modified', models.DateTimeField(auto_now=True, null=True)), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('file_size', models.PositiveIntegerField(default=0)), + ], + options={ + 'ordering': ('pk',), + }, + ), + migrations.CreateModel( + name='MergedXForm', + fields=[ + ('xform_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='logger.xform')), + ('xforms', models.ManyToManyField(related_name='mergedxform_ptr', to='logger.XForm')), + ], + options={ + 'permissions': (('view_mergedxform', 'Can view associated data'),), + }, + bases=('logger.xform',), + ), + migrations.CreateModel( + name='InstanceHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xml', models.TextField()), + ('uuid', models.CharField(default='', max_length=249)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('xform_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='submission_history', to='logger.instance')), + ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(null=True, srid=4326)), + ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('submission_date', models.DateTimeField(default=None, null=True)), + ('checksum', models.CharField(blank=True, max_length=64, null=True)), + ], + ), + migrations.RunSQL( + sql="UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", + reverse_sql='', + ), + migrations.AddField( + model_name='attachment', + name='name', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='xform', + name='uuid', + field=models.CharField(default='', max_length=36), + ), + migrations.AddField( + model_name='dataview', + name='deleted_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='dataview', + name='deleted_by', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataview_deleted_by', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='xform', + name='deleted_by', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='project', + name='deleted_by', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_deleted_by', to=settings.AUTH_USER_MODEL), + ), + migrations.RunPython( + recalculate_xform_hash + ), + migrations.AddField( + model_name='instance', + name='deleted_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='instance', + name='has_a_review', + field=models.BooleanField(default=False, verbose_name='has_a_review'), + ), + migrations.CreateModel( + name='SubmissionReview', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('1', 'Approved'), ('3', 'Pending'), ('2', 'Rejected')], db_index=True, default='3', max_length=1, verbose_name='Status')), + ('deleted_at', models.DateTimeField(db_index=True, default=None, null=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('created_by', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('deleted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_reviews', to=settings.AUTH_USER_MODEL)), + ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='reviews', to='logger.instance')), + ('note', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='notes', to='logger.note')), + ], + ), + migrations.AlterModelOptions( + name='mergedxform', + options={}, + ), + migrations.AlterModelOptions( + name='note', + options={}, + ), + migrations.AlterModelOptions( + name='project', + options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, + ), + migrations.AlterModelOptions( + name='xform', + options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + ), + migrations.AlterField( + model_name='attachment', + name='mimetype', + field=models.CharField(blank=True, default='', max_length=100), + ), + migrations.AlterField( + model_name='instance', + name='survey_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.surveytype'), + ), + migrations.AlterField( + model_name='instance', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='osmdata', + name='field_name', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='osmdata', + name='osm_type', + field=models.CharField(default='way', max_length=10), + ), + migrations.AlterField( + model_name='project', + name='tags', + field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + ), + migrations.AlterField( + model_name='widget', + name='order', + field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + ), + migrations.AlterField( + model_name='widget', + name='widget_type', + field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), + ), + migrations.AlterField( + model_name='xform', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='xform', + name='sms_id_string', + field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), + ), + migrations.AddField( + model_name='xform', + name='public_key', + field=models.TextField(blank=True, default='', null=True), + ), + migrations.AddField( + model_name='attachment', + name='deleted_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='xform', + name='uuid', + field=models.CharField(db_index=True, default='', max_length=36), + ), + migrations.RunPython(generate_uuid_if_missing), + migrations.RunPython(regenerate_instance_json), + migrations.CreateModel( + name='XFormVersion', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xls', models.FileField(upload_to='')), + ('version', models.CharField(max_length=100)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('xml', models.TextField()), + ('json', models.TextField()), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.xform')), + ], + options={ + 'unique_together': {('xform', 'version')}, + }, + ), + migrations.RunPython(create_initial_xform_version), + ] From 562a50760c058001900c0c40fe6e3cb623f75413 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:15:04 +0300 Subject: [PATCH 023/234] Alter JSON column field types to use `django.db.models.JSONField` --- .../migrations/0002_auto_20220425_0313.py | 54 +++++++++++++++++++ .../migrations/0010_auto_20220425_0313.py | 23 ++++++++ .../migrations/0009_alter_export_options.py | 18 +++++++ 3 files changed, 95 insertions(+) create mode 100644 onadata/apps/logger/migrations/0002_auto_20220425_0313.py create mode 100644 onadata/apps/main/migrations/0010_auto_20220425_0313.py create mode 100644 onadata/apps/viewer/migrations/0009_alter_export_options.py diff --git a/onadata/apps/logger/migrations/0002_auto_20220425_0313.py b/onadata/apps/logger/migrations/0002_auto_20220425_0313.py new file mode 100644 index 0000000000..00f5a7727d --- /dev/null +++ b/onadata/apps/logger/migrations/0002_auto_20220425_0313.py @@ -0,0 +1,54 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0001_pre-django-3-upgrade'), + ] + + operations = [ + migrations.AlterField( + model_name='dataview', + name='columns', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='dataview', + name='query', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='instance', + name='json', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='instance', + name='xform', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform'), + ), + migrations.AlterField( + model_name='osmdata', + name='tags', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='project', + name='metadata', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='widget', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='xform', + name='last_updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/onadata/apps/main/migrations/0010_auto_20220425_0313.py b/onadata/apps/main/migrations/0010_auto_20220425_0313.py new file mode 100644 index 0000000000..404cfa67a9 --- /dev/null +++ b/onadata/apps/main/migrations/0010_auto_20220425_0313.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0009_auto_20190125_0517'), + ] + + operations = [ + migrations.AlterField( + model_name='audit', + name='json', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='userprofile', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/onadata/apps/viewer/migrations/0009_alter_export_options.py b/onadata/apps/viewer/migrations/0009_alter_export_options.py new file mode 100644 index 0000000000..d703c213fd --- /dev/null +++ b/onadata/apps/viewer/migrations/0009_alter_export_options.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0008_auto_20190125_0517'), + ] + + operations = [ + migrations.AlterField( + model_name='export', + name='options', + field=models.JSONField(default=dict), + ), + ] From ea08db058438090c76193eefadb498225bf67bc0 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:20:03 +0300 Subject: [PATCH 024/234] Use `JSONField` field type instead of `TextField` on the XForm model --- .../logger/migrations/0003_alter_xform_json.py | 18 ++++++++++++++++++ onadata/apps/logger/models/xform.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 onadata/apps/logger/migrations/0003_alter_xform_json.py diff --git a/onadata/apps/logger/migrations/0003_alter_xform_json.py b/onadata/apps/logger/migrations/0003_alter_xform_json.py new file mode 100644 index 0000000000..16266d0f99 --- /dev/null +++ b/onadata/apps/logger/migrations/0003_alter_xform_json.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0002_auto_20220425_0313'), + ] + + operations = [ + migrations.AlterField( + model_name='xform', + name='json', + field=models.JSONField(default=dict), + ), + ] diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index d7b88d0377..a47631e901 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -738,7 +738,7 @@ class XForm(XFormMixin, BaseModel): MAX_ID_LENGTH = 100 xls = models.FileField(upload_to=upload_to, null=True) - json = models.TextField(default="") + json = models.JSONField(default=dict) description = models.TextField(default="", null=True, blank=True) xml = models.TextField() From ad3fe9c6f23064f0d39f01127c24b005c3a2bad4 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 10:25:37 +0300 Subject: [PATCH 025/234] Add DEFAULT_AUTO_FIELD Signed-off-by: Kipchirchir Sigei --- onadata/settings/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 535b7d466e..3ade3e2e9b 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -152,6 +152,7 @@ }, ] +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' MIDDLEWARE = ( "onadata.libs.profiling.sql.SqlTimingMiddleware", From a96f8a2c20de860bf9a57703af2ff0fa250c7a06 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:29:13 +0300 Subject: [PATCH 026/234] Transition to using the squashed migration - Remove old migration files - Remove `replaces` attribute on the Squashed migration file --- .../apps/logger/migrations/0001_initial.py | 339 ------------------ .../migrations/0001_pre-django-3-upgrade.py | 3 - .../migrations/0002_auto_20150717_0048.py | 32 -- .../migrations/0002_auto_20220425_0313.py | 54 --- .../migrations/0003_alter_xform_json.py | 18 - .../0003_dataview_instances_with_geopoints.py | 20 -- .../migrations/0004_auto_20150910_0056.py | 59 --- .../migrations/0005_auto_20151015_0758.py | 59 --- .../migrations/0006_auto_20151106_0130.py | 53 --- .../migrations/0007_osmdata_field_name.py | 20 -- .../migrations/0008_osmdata_osm_type.py | 20 -- .../migrations/0009_auto_20151111_0438.py | 18 - .../migrations/0010_attachment_file_size.py | 20 -- .../0011_dataview_matches_parent.py | 20 -- .../migrations/0012_auto_20160114_0708.py | 20 -- .../logger/migrations/0013_note_created_by.py | 23 -- .../migrations/0014_note_instance_field.py | 20 -- .../migrations/0015_auto_20160222_0559.py | 25 -- .../migrations/0016_widget_aggregation.py | 21 -- .../migrations/0017_auto_20160224_0130.py | 32 -- .../migrations/0018_auto_20160301_0330.py | 32 -- .../migrations/0019_auto_20160307_0256.py | 35 -- .../migrations/0020_auto_20160408_0325.py | 48 --- .../migrations/0021_auto_20160408_0919.py | 21 -- .../migrations/0022_auto_20160418_0518.py | 25 -- .../migrations/0023_auto_20160419_0403.py | 26 -- .../migrations/0024_xform_has_hxl_support.py | 20 -- .../migrations/0025_xform_last_updated_at.py | 26 -- .../migrations/0026_auto_20160913_0239.py | 20 -- .../migrations/0027_auto_20161201_0730.py | 21 -- .../migrations/0028_auto_20170217_0502.py | 52 --- .../migrations/0028_auto_20170221_0838.py | 34 -- .../migrations/0029_auto_20170221_0908.py | 22 -- .../migrations/0030_auto_20170227_0137.py | 23 -- onadata/apps/logger/migrations/0031_merge.py | 16 - .../migrations/0032_project_deleted_at.py | 20 -- .../migrations/0033_auto_20170705_0159.py | 35 -- .../migrations/0034_auto_20170814_0432.py | 39 -- .../logger/migrations/0034_mergedxform.py | 28 -- .../migrations/0035_auto_20170712_0529.py | 19 - .../0036_xform_is_merged_dataset.py | 20 -- .../migrations/0037_merge_20170825_0238.py | 16 - .../migrations/0038_auto_20170828_1718.py | 33 -- .../migrations/0039_auto_20170909_2052.py | 25 -- .../migrations/0040_auto_20170912_1504.py | 20 -- .../migrations/0041_auto_20170912_1512.py | 20 -- .../apps/logger/migrations/0042_xform_hash.py | 21 -- .../migrations/0043_auto_20171010_0403.py | 21 -- .../migrations/0044_xform_hash_sql_update.py | 18 - .../logger/migrations/0045_attachment_name.py | 20 -- .../migrations/0046_auto_20180314_1618.py | 20 -- .../migrations/0047_dataview_deleted_at.py | 20 -- .../migrations/0048_dataview_deleted_by.py | 29 -- .../migrations/0049_xform_deleted_by.py | 28 -- .../migrations/0050_project_deleted_by.py | 27 -- .../migrations/0051_auto_20180522_1118.py | 45 --- .../migrations/0052_auto_20180805_2233.py | 30 -- .../migrations/0053_submissionreview.py | 66 ---- .../migrations/0054_instance_has_a_review.py | 20 -- .../migrations/0055_auto_20180904_0713.py | 20 -- .../migrations/0056_auto_20190125_0517.py | 87 ----- .../migrations/0057_xform_public_key.py | 18 - .../migrations/0058_auto_20191211_0900.py | 18 - .../migrations/0059_attachment_deleted_by.py | 21 -- .../migrations/0060_auto_20200305_0357.py | 18 - .../migrations/0061_auto_20200713_0814.py | 26 -- .../migrations/0062_auto_20210202_0248.py | 28 -- .../logger/migrations/0063_xformversion.py | 33 -- .../migrations/0064_auto_20210304_0314.py | 31 -- 69 files changed, 2237 deletions(-) delete mode 100644 onadata/apps/logger/migrations/0001_initial.py delete mode 100644 onadata/apps/logger/migrations/0002_auto_20150717_0048.py delete mode 100644 onadata/apps/logger/migrations/0002_auto_20220425_0313.py delete mode 100644 onadata/apps/logger/migrations/0003_alter_xform_json.py delete mode 100644 onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py delete mode 100644 onadata/apps/logger/migrations/0004_auto_20150910_0056.py delete mode 100644 onadata/apps/logger/migrations/0005_auto_20151015_0758.py delete mode 100644 onadata/apps/logger/migrations/0006_auto_20151106_0130.py delete mode 100644 onadata/apps/logger/migrations/0007_osmdata_field_name.py delete mode 100644 onadata/apps/logger/migrations/0008_osmdata_osm_type.py delete mode 100644 onadata/apps/logger/migrations/0009_auto_20151111_0438.py delete mode 100644 onadata/apps/logger/migrations/0010_attachment_file_size.py delete mode 100644 onadata/apps/logger/migrations/0011_dataview_matches_parent.py delete mode 100644 onadata/apps/logger/migrations/0012_auto_20160114_0708.py delete mode 100644 onadata/apps/logger/migrations/0013_note_created_by.py delete mode 100644 onadata/apps/logger/migrations/0014_note_instance_field.py delete mode 100644 onadata/apps/logger/migrations/0015_auto_20160222_0559.py delete mode 100644 onadata/apps/logger/migrations/0016_widget_aggregation.py delete mode 100644 onadata/apps/logger/migrations/0017_auto_20160224_0130.py delete mode 100644 onadata/apps/logger/migrations/0018_auto_20160301_0330.py delete mode 100644 onadata/apps/logger/migrations/0019_auto_20160307_0256.py delete mode 100644 onadata/apps/logger/migrations/0020_auto_20160408_0325.py delete mode 100644 onadata/apps/logger/migrations/0021_auto_20160408_0919.py delete mode 100644 onadata/apps/logger/migrations/0022_auto_20160418_0518.py delete mode 100644 onadata/apps/logger/migrations/0023_auto_20160419_0403.py delete mode 100644 onadata/apps/logger/migrations/0024_xform_has_hxl_support.py delete mode 100644 onadata/apps/logger/migrations/0025_xform_last_updated_at.py delete mode 100644 onadata/apps/logger/migrations/0026_auto_20160913_0239.py delete mode 100644 onadata/apps/logger/migrations/0027_auto_20161201_0730.py delete mode 100644 onadata/apps/logger/migrations/0028_auto_20170217_0502.py delete mode 100644 onadata/apps/logger/migrations/0028_auto_20170221_0838.py delete mode 100644 onadata/apps/logger/migrations/0029_auto_20170221_0908.py delete mode 100644 onadata/apps/logger/migrations/0030_auto_20170227_0137.py delete mode 100644 onadata/apps/logger/migrations/0031_merge.py delete mode 100644 onadata/apps/logger/migrations/0032_project_deleted_at.py delete mode 100644 onadata/apps/logger/migrations/0033_auto_20170705_0159.py delete mode 100644 onadata/apps/logger/migrations/0034_auto_20170814_0432.py delete mode 100644 onadata/apps/logger/migrations/0034_mergedxform.py delete mode 100644 onadata/apps/logger/migrations/0035_auto_20170712_0529.py delete mode 100644 onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py delete mode 100644 onadata/apps/logger/migrations/0037_merge_20170825_0238.py delete mode 100644 onadata/apps/logger/migrations/0038_auto_20170828_1718.py delete mode 100644 onadata/apps/logger/migrations/0039_auto_20170909_2052.py delete mode 100644 onadata/apps/logger/migrations/0040_auto_20170912_1504.py delete mode 100644 onadata/apps/logger/migrations/0041_auto_20170912_1512.py delete mode 100644 onadata/apps/logger/migrations/0042_xform_hash.py delete mode 100644 onadata/apps/logger/migrations/0043_auto_20171010_0403.py delete mode 100644 onadata/apps/logger/migrations/0044_xform_hash_sql_update.py delete mode 100644 onadata/apps/logger/migrations/0045_attachment_name.py delete mode 100644 onadata/apps/logger/migrations/0046_auto_20180314_1618.py delete mode 100644 onadata/apps/logger/migrations/0047_dataview_deleted_at.py delete mode 100644 onadata/apps/logger/migrations/0048_dataview_deleted_by.py delete mode 100644 onadata/apps/logger/migrations/0049_xform_deleted_by.py delete mode 100644 onadata/apps/logger/migrations/0050_project_deleted_by.py delete mode 100644 onadata/apps/logger/migrations/0051_auto_20180522_1118.py delete mode 100644 onadata/apps/logger/migrations/0052_auto_20180805_2233.py delete mode 100644 onadata/apps/logger/migrations/0053_submissionreview.py delete mode 100644 onadata/apps/logger/migrations/0054_instance_has_a_review.py delete mode 100644 onadata/apps/logger/migrations/0055_auto_20180904_0713.py delete mode 100644 onadata/apps/logger/migrations/0056_auto_20190125_0517.py delete mode 100644 onadata/apps/logger/migrations/0057_xform_public_key.py delete mode 100644 onadata/apps/logger/migrations/0058_auto_20191211_0900.py delete mode 100644 onadata/apps/logger/migrations/0059_attachment_deleted_by.py delete mode 100644 onadata/apps/logger/migrations/0060_auto_20200305_0357.py delete mode 100644 onadata/apps/logger/migrations/0061_auto_20200713_0814.py delete mode 100644 onadata/apps/logger/migrations/0062_auto_20210202_0248.py delete mode 100644 onadata/apps/logger/migrations/0063_xformversion.py delete mode 100644 onadata/apps/logger/migrations/0064_auto_20210304_0314.py diff --git a/onadata/apps/logger/migrations/0001_initial.py b/onadata/apps/logger/migrations/0001_initial.py deleted file mode 100644 index e1d3f4fbc8..0000000000 --- a/onadata/apps/logger/migrations/0001_initial.py +++ /dev/null @@ -1,339 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import jsonfield.fields -import onadata.apps.logger.models.xform -import django.contrib.gis.db.models.fields -from django.conf import settings -import onadata.apps.logger.models.attachment -import taggit.managers - - -class Migration(migrations.Migration): - - dependencies = [ - ('taggit', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('contenttypes', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Attachment', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('media_file', models.FileField( - max_length=255, - upload_to=onadata.apps.logger.models.attachment.upload_to) - ), - ('mimetype', models.CharField( - default=b'', max_length=50, blank=True)), - ('extension', - models.CharField(default='non', max_length=10, db_index=True) - ), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='DataView', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('columns', jsonfield.fields.JSONField()), - ('query', jsonfield.fields.JSONField(default={}, blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ], - options={ - 'verbose_name': 'Data View', - 'verbose_name_plural': 'Data Views', - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='Instance', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('json', jsonfield.fields.JSONField(default={})), - ('xml', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('status', models.CharField(default='submitted_via_web', - max_length=20)), - ('uuid', models.CharField(default='', max_length=249)), - ('version', models.CharField(max_length=255, null=True)), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='InstanceHistory', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xml', models.TextField()), - ('uuid', models.CharField(default='', max_length=249)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xform_instance', - models.ForeignKey(related_name='submission_history', - to='logger.Instance', - on_delete=models.CASCADE)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='Note', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('note', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('instance', - models.ForeignKey( - related_name='notes', to='logger.Instance', - on_delete=models.CASCADE) - ), - ], - options={ - 'permissions': (('view_note', 'View note'),), - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='Project', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('metadata', jsonfield.fields.JSONField(blank=True)), - ('shared', models.BooleanField(default=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey(related_name='project_owner', - to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('organization', models.ForeignKey( - related_name='project_org', to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user_stars', - models.ManyToManyField(related_name='project_stars', - to=settings.AUTH_USER_MODEL)), - ], - options={ - 'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project')), - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='SurveyType', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('slug', models.CharField(unique=True, max_length=100)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='Widget', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('object_id', models.PositiveIntegerField()), - ('widget_type', - models.CharField(default=b'charts', max_length=25, - choices=[(b'charts', b'Charts')])), - ('view_type', models.CharField(max_length=50)), - ('column', models.CharField(max_length=50)), - ('group_by', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('title', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('description', - models.CharField(default=None, max_length=255, - null=True, blank=True)), - ('key', - models.CharField(unique=True, max_length=32, db_index=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', - models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='XForm', - fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xls', - models.FileField( - null=True, - upload_to=onadata.apps.logger.models.xform.upload_to)), - ('json', models.TextField(default='')), - ('description', models.TextField(default='', null=True, - blank=True)), - ('xml', models.TextField()), - ('require_auth', models.BooleanField(default=False)), - ('shared', models.BooleanField(default=False)), - ('shared_data', models.BooleanField(default=False)), - ('downloadable', models.BooleanField(default=True)), - ('allows_sms', models.BooleanField(default=False)), - ('encrypted', models.BooleanField(default=False)), - ('sms_id_string', - models.SlugField(default=b'', verbose_name='SMS ID', - max_length=100, editable=False)), - ('id_string', - models.SlugField(verbose_name='ID', max_length=100, - editable=False)), - ('title', models.CharField(max_length=255, editable=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(null=True, blank=True)), - ('last_submission_time', - models.DateTimeField(null=True, blank=True)), - ('has_start_time', models.BooleanField(default=False)), - ('uuid', models.CharField(default='', max_length=32)), - ('bamboo_dataset', - models.CharField(default='', max_length=60)), - ('instances_with_geopoints', - models.BooleanField(default=False)), - ('instances_with_osm', models.BooleanField(default=False)), - ('num_of_submissions', models.IntegerField(default=0)), - ('version', - models.CharField(max_length=255, null=True, blank=True)), - ('created_by', - models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE)), - ('project', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user', - models.ForeignKey(related_name='xforms', - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE)) - ], - options={ - 'ordering': ('id_string',), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data')), - }, - bases=(models.Model,), - ), - migrations.AlterUniqueTogether( - name='xform', - unique_together=set( - [('user', 'id_string', 'project'), - ('user', 'sms_id_string', 'project')]), - ), - migrations.AlterUniqueTogether( - name='project', - unique_together=set([('name', 'organization')]), - ), - migrations.AddField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(to='logger.SurveyType', - on_delete=models.CASCADE), - preserve_default=True, - ), - migrations.AddField( - model_name='instance', - name='tags', - field=taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags'), - preserve_default=True, - ), - migrations.AddField( - model_name='instance', - name='user', - field=models.ForeignKey( - related_name='instances', to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE, - null=True), - preserve_default=True, - ), - migrations.AddField( - model_name='instance', - name='xform', - field=models.ForeignKey( - related_name='instances', - to='logger.XForm', null=True, on_delete=models.CASCADE), - preserve_default=True, - ), - migrations.AddField( - model_name='dataview', - name='project', - field=models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE), - preserve_default=True, - ), - migrations.AddField( - model_name='dataview', - name='xform', - field=models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE), - preserve_default=True, - ), - migrations.AddField( - model_name='attachment', - name='instance', - field=models.ForeignKey( - related_name='attachments',to='logger.Instance', - on_delete=models.CASCADE), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py index 8c18800737..1ea9f64186 100644 --- a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py +++ b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py @@ -76,9 +76,6 @@ def create_initial_xform_version(apps, schema_editor): class Migration(migrations.Migration): - - replaces = [('logger', '0001_initial'), ('logger', '0002_auto_20150717_0048'), ('logger', '0003_dataview_instances_with_geopoints'), ('logger', '0004_auto_20150910_0056'), ('logger', '0005_auto_20151015_0758'), ('logger', '0006_auto_20151106_0130'), ('logger', '0007_osmdata_field_name'), ('logger', '0008_osmdata_osm_type'), ('logger', '0009_auto_20151111_0438'), ('logger', '0010_attachment_file_size'), ('logger', '0011_dataview_matches_parent'), ('logger', '0012_auto_20160114_0708'), ('logger', '0013_note_created_by'), ('logger', '0014_note_instance_field'), ('logger', '0015_auto_20160222_0559'), ('logger', '0016_widget_aggregation'), ('logger', '0017_auto_20160224_0130'), ('logger', '0018_auto_20160301_0330'), ('logger', '0019_auto_20160307_0256'), ('logger', '0020_auto_20160408_0325'), ('logger', '0021_auto_20160408_0919'), ('logger', '0022_auto_20160418_0518'), ('logger', '0023_auto_20160419_0403'), ('logger', '0024_xform_has_hxl_support'), ('logger', '0025_xform_last_updated_at'), ('logger', '0026_auto_20160913_0239'), ('logger', '0027_auto_20161201_0730'), ('logger', '0028_auto_20170221_0838'), ('logger', '0029_auto_20170221_0908'), ('logger', '0030_auto_20170227_0137'), ('logger', '0028_auto_20170217_0502'), ('logger', '0031_merge'), ('logger', '0032_project_deleted_at'), ('logger', '0033_auto_20170705_0159'), ('logger', '0034_mergedxform'), ('logger', '0035_auto_20170712_0529'), ('logger', '0036_xform_is_merged_dataset'), ('logger', '0034_auto_20170814_0432'), ('logger', '0037_merge_20170825_0238'), ('logger', '0038_auto_20170828_1718'), ('logger', '0039_auto_20170909_2052'), ('logger', '0040_auto_20170912_1504'), ('logger', '0041_auto_20170912_1512'), ('logger', '0042_xform_hash'), ('logger', '0043_auto_20171010_0403'), ('logger', '0044_xform_hash_sql_update'), ('logger', '0045_attachment_name'), ('logger', '0046_auto_20180314_1618'), ('logger', '0047_dataview_deleted_at'), ('logger', '0048_dataview_deleted_by'), ('logger', '0049_xform_deleted_by'), ('logger', '0050_project_deleted_by'), ('logger', '0051_auto_20180522_1118'), ('logger', '0052_auto_20180805_2233'), ('logger', '0053_submissionreview'), ('logger', '0054_instance_has_a_review'), ('logger', '0055_auto_20180904_0713'), ('logger', '0056_auto_20190125_0517'), ('logger', '0057_xform_public_key'), ('logger', '0058_auto_20191211_0900'), ('logger', '0059_attachment_deleted_by'), ('logger', '0060_auto_20200305_0357'), ('logger', '0061_auto_20200713_0814'), ('logger', '0062_auto_20210202_0248'), ('logger', '0063_xformversion'), ('logger', '0064_auto_20210304_0314')] - initial = True dependencies = [ diff --git a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py deleted file mode 100644 index 4ae94dca71..0000000000 --- a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='attachment', - name='date_created', - field=models.DateTimeField(auto_now_add=True, null=True), - preserve_default=True, - ), - migrations.AddField( - model_name='attachment', - name='date_modified', - field=models.DateTimeField(auto_now=True, null=True), - preserve_default=True, - ), - migrations.AddField( - model_name='attachment', - name='deleted_at', - field=models.DateTimeField(default=None, null=True), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0002_auto_20220425_0313.py b/onadata/apps/logger/migrations/0002_auto_20220425_0313.py deleted file mode 100644 index 00f5a7727d..0000000000 --- a/onadata/apps/logger/migrations/0002_auto_20220425_0313.py +++ /dev/null @@ -1,54 +0,0 @@ -# Generated by Django 3.2.13 on 2022-04-25 07:13 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), - ] - - operations = [ - migrations.AlterField( - model_name='dataview', - name='columns', - field=models.JSONField(), - ), - migrations.AlterField( - model_name='dataview', - name='query', - field=models.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='instance', - name='json', - field=models.JSONField(default=dict), - ), - migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform'), - ), - migrations.AlterField( - model_name='osmdata', - name='tags', - field=models.JSONField(default=dict), - ), - migrations.AlterField( - model_name='project', - name='metadata', - field=models.JSONField(default=dict), - ), - migrations.AlterField( - model_name='widget', - name='metadata', - field=models.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='xform', - name='last_updated_at', - field=models.DateTimeField(auto_now=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0003_alter_xform_json.py b/onadata/apps/logger/migrations/0003_alter_xform_json.py deleted file mode 100644 index 16266d0f99..0000000000 --- a/onadata/apps/logger/migrations/0003_alter_xform_json.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.13 on 2022-04-25 07:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0002_auto_20220425_0313'), - ] - - operations = [ - migrations.AlterField( - model_name='xform', - name='json', - field=models.JSONField(default=dict), - ), - ] diff --git a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py deleted file mode 100644 index af36fdd965..0000000000 --- a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0002_auto_20150717_0048'), - ] - - operations = [ - migrations.AddField( - model_name='dataview', - name='instances_with_geopoints', - field=models.BooleanField(default=False), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py deleted file mode 100644 index 5949179146..0000000000 --- a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0003_dataview_instances_with_geopoints'), - ] - - operations = [ - migrations.CreateModel( - name='ProjectGroupObjectPermission', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey( - to='logger.Project', on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ], - options={ - 'abstract': False, - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='ProjectUserObjectPermission', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ], - options={ - 'abstract': False, - }, - bases=(models.Model,), - ), - migrations.AlterUniqueTogether( - name='projectuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), - ), - migrations.AlterUniqueTogether( - name='projectgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), - ), - ] diff --git a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py deleted file mode 100644 index db63190a51..0000000000 --- a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0004_auto_20150910_0056'), - ] - - operations = [ - migrations.CreateModel( - name='XFormGroupObjectPermission', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ], - options={ - 'abstract': False, - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name='XFormUserObjectPermission', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ], - options={ - 'abstract': False, - }, - bases=(models.Model,), - ), - migrations.AlterUniqueTogether( - name='xformuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), - ), - migrations.AlterUniqueTogether( - name='xformgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), - ), - ] diff --git a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py deleted file mode 100644 index aa0f1b0137..0000000000 --- a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import jsonfield.fields -import django.contrib.gis.db.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0005_auto_20151015_0758'), - ] - - operations = [ - migrations.CreateModel( - name='OsmData', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('xml', models.TextField()), - ('osm_id', models.CharField(max_length=10)), - ('tags', jsonfield.fields.JSONField(default={})), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326)), - ('filename', models.CharField(max_length=255)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('instance', models.ForeignKey(related_name='osm_data', - to='logger.Instance', - on_delete=models.CASCADE)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.AlterModelOptions( - name='xform', - options={ - 'ordering': ('id_string',), 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form') - )}, - ), - ] diff --git a/onadata/apps/logger/migrations/0007_osmdata_field_name.py b/onadata/apps/logger/migrations/0007_osmdata_field_name.py deleted file mode 100644 index 7870df2c4e..0000000000 --- a/onadata/apps/logger/migrations/0007_osmdata_field_name.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0006_auto_20151106_0130'), - ] - - operations = [ - migrations.AddField( - model_name='osmdata', - name='field_name', - field=models.CharField(default=b'', max_length=255, blank=True), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py deleted file mode 100644 index 7bde859ecf..0000000000 --- a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0007_osmdata_field_name'), - ] - - operations = [ - migrations.AddField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default=b'way', max_length=10), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py deleted file mode 100644 index 96113b0393..0000000000 --- a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0008_osmdata_osm_type'), - ] - - operations = [ - migrations.AlterUniqueTogether( - name='osmdata', - unique_together=set([('instance', 'field_name')]), - ), - ] diff --git a/onadata/apps/logger/migrations/0010_attachment_file_size.py b/onadata/apps/logger/migrations/0010_attachment_file_size.py deleted file mode 100644 index 63bcc1c617..0000000000 --- a/onadata/apps/logger/migrations/0010_attachment_file_size.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0009_auto_20151111_0438'), - ] - - operations = [ - migrations.AddField( - model_name='attachment', - name='file_size', - field=models.PositiveIntegerField(default=0), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py deleted file mode 100644 index cee1b46ea5..0000000000 --- a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0010_attachment_file_size'), - ] - - operations = [ - migrations.AddField( - model_name='dataview', - name='matches_parent', - field=models.BooleanField(default=False), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py deleted file mode 100644 index ccbae70aaa..0000000000 --- a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0011_dataview_matches_parent'), - ] - - operations = [ - migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(default=b'', max_length=100, blank=True), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0013_note_created_by.py b/onadata/apps/logger/migrations/0013_note_created_by.py deleted file mode 100644 index eae6d40b75..0000000000 --- a/onadata/apps/logger/migrations/0013_note_created_by.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0012_auto_20160114_0708'), - ] - - operations = [ - migrations.AddField( - model_name='note', - name='created_by', - field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0014_note_instance_field.py b/onadata/apps/logger/migrations/0014_note_instance_field.py deleted file mode 100644 index 8db30ebdae..0000000000 --- a/onadata/apps/logger/migrations/0014_note_instance_field.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0013_note_created_by'), - ] - - operations = [ - migrations.AddField( - model_name='note', - name='instance_field', - field=models.TextField(null=True, blank=True), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py deleted file mode 100644 index eb6cbcccd4..0000000000 --- a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0014_note_instance_field'), - ] - - operations = [ - migrations.AlterModelOptions( - name='widget', - options={'ordering': ('order',)}, - ), - migrations.AddField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(default=0, editable=False, - db_index=True), - preserve_default=False, - ), - ] diff --git a/onadata/apps/logger/migrations/0016_widget_aggregation.py b/onadata/apps/logger/migrations/0016_widget_aggregation.py deleted file mode 100644 index d429a246ea..0000000000 --- a/onadata/apps/logger/migrations/0016_widget_aggregation.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0015_auto_20160222_0559'), - ] - - operations = [ - migrations.AddField( - model_name='widget', - name='aggregation', - field=models.CharField(default=None, max_length=255, null=True, - blank=True), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py deleted file mode 100644 index f6e71f7031..0000000000 --- a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0016_widget_aggregation'), - ] - - operations = [ - migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(max_length=249), - preserve_default=True, - ), - migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', default=-1, - to='logger.XForm', - on_delete=models.CASCADE), - preserve_default=False, - ), - migrations.AlterUniqueTogether( - name='instance', - unique_together=set([('xform', 'uuid')]), - ), - ] diff --git a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py deleted file mode 100644 index 077a7dce24..0000000000 --- a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings -import django.contrib.gis.db.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0017_auto_20160224_0130'), - ] - - operations = [ - migrations.AddField( - model_name='instancehistory', - name='geom', - field=django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True), - preserve_default=True, - ), - migrations.AddField( - model_name='instancehistory', - name='user', - field=models.ForeignKey( - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py deleted file mode 100644 index bbb5f3bdfd..0000000000 --- a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import jsonfield.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0018_auto_20160301_0330'), - ] - - operations = [ - migrations.AddField( - model_name='widget', - name='metadata', - field=jsonfield.fields.JSONField(default={}, blank=True), - preserve_default=True, - ), - migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(default='', max_length=249), - preserve_default=True, - ), - migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', - to='logger.XForm', null=True, - on_delete=models.CASCADE), - preserve_default=True, - ), - ] diff --git a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py deleted file mode 100644 index 9353329eae..0000000000 --- a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-08 07:25 -from __future__ import unicode_literals - -import django.contrib.postgres.fields.jsonb -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0019_auto_20160307_0256'), - ] - - operations = [ - migrations.AlterField( - model_name='dataview', - name='columns', - field=django.contrib.postgres.fields.jsonb.JSONField(), - ), - migrations.AlterField( - model_name='dataview', - name='query', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), - ), - migrations.AlterField( - model_name='instance', - name='json', - field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), - ), - migrations.AlterField( - model_name='osmdata', - name='tags', - field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), - ), - migrations.AlterField( - model_name='project', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True), - ), - migrations.AlterField( - model_name='widget', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), - ), - ] diff --git a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py deleted file mode 100644 index dbe27f3f87..0000000000 --- a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-08 13:19 -from __future__ import unicode_literals - -import django.contrib.postgres.fields.jsonb -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0020_auto_20160408_0325'), - ] - - operations = [ - migrations.AlterField( - model_name='project', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), - ), - ] diff --git a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py deleted file mode 100644 index 8954428f43..0000000000 --- a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-18 09:18 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0021_auto_20160408_0919'), - ] - - operations = [ - migrations.AddField( - model_name='instance', - name='last_edited', - field=models.DateTimeField(default=None, null=True), - ), - migrations.AddField( - model_name='instancehistory', - name='submission_date', - field=models.DateTimeField(default=None, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py deleted file mode 100644 index dc99e5bdb0..0000000000 --- a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-19 08:03 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0022_auto_20160418_0518'), - ] - - operations = [ - migrations.AlterField( - model_name='widget', - name='column', - field=models.CharField(max_length=255), - ), - migrations.AlterField( - model_name='widget', - name='group_by', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py deleted file mode 100644 index 64a4ad8912..0000000000 --- a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-19 09:54 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0023_auto_20160419_0403'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='has_hxl_support', - field=models.BooleanField(default=False), - ), - ] diff --git a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py deleted file mode 100644 index f8000e7012..0000000000 --- a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-08-18 12:43 -from __future__ import unicode_literals - -import datetime -from django.db import migrations, models -from django.utils.timezone import utc - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0024_xform_has_hxl_support'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='last_updated_at', - field=models.DateTimeField( - auto_now=True, - default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, - tzinfo=utc)), - preserve_default=False, - ), - ] diff --git a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py deleted file mode 100644 index d29b823400..0000000000 --- a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-09-13 06:39 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0025_xform_last_updated_at'), - ] - - operations = [ - migrations.AlterField( - model_name='osmdata', - name='osm_id', - field=models.CharField(max_length=20), - ), - ] diff --git a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py deleted file mode 100644 index d253875e2c..0000000000 --- a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-12-01 12:30 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0026_auto_20160913_0239'), - ] - - operations = [ - migrations.AlterField( - model_name='widget', - name='title', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py deleted file mode 100644 index 39734a6f06..0000000000 --- a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-02-17 10:02 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0027_auto_20161201_0730'), - ] - - operations = [ - migrations.AlterModelOptions( - name='project', - options={'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project'), - ('view_project_all', 'Can view all associated data'), - ('view_project_data', 'Can view submitted data'))}, - ), - migrations.AlterModelOptions( - name='xform', - options={ - 'ordering': ('id_string',), - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, - ), - migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey( - default=328, on_delete=django.db.models.deletion.CASCADE, - related_name='instances', to='logger.XForm'), - preserve_default=False, - ), - ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py deleted file mode 100644 index 07c68426c3..0000000000 --- a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-02-21 13:38 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('logger', '0027_auto_20161201_0730'), - ] - - operations = [ - migrations.CreateModel( - name='OpenData', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, - serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('uuid', models.CharField(default='', max_length=32)), - ('object_id', models.PositiveIntegerField(blank=True, - null=True)), - ('active', models.BooleanField(default=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to='contenttypes.ContentType')), - ], - ), - ] diff --git a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py deleted file mode 100644 index 681b1890e9..0000000000 --- a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-02-21 14:08 -from __future__ import unicode_literals - -from django.db import migrations, models -import onadata.apps.logger.models.open_data - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0028_auto_20170221_0838'), - ] - - operations = [ - migrations.AlterField( - model_name='opendata', - name='uuid', - field=models.CharField( - default=onadata.apps.logger.models.open_data.get_uuid, - max_length=32), ), - ] diff --git a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py deleted file mode 100644 index a4e879564f..0000000000 --- a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-02-27 06:37 -from __future__ import unicode_literals - -from django.db import migrations, models -import onadata.libs.utils.common_tools - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0029_auto_20170221_0908'), - ] - - operations = [ - migrations.AlterField( - model_name='opendata', - name='uuid', - field=models.CharField( - default=onadata.libs.utils.common_tools.get_uuid, - max_length=32, - unique=True), ), - ] diff --git a/onadata/apps/logger/migrations/0031_merge.py b/onadata/apps/logger/migrations/0031_merge.py deleted file mode 100644 index 3d24ace1f5..0000000000 --- a/onadata/apps/logger/migrations/0031_merge.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-02-28 08:00 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0028_auto_20170217_0502'), - ('logger', '0030_auto_20170227_0137'), - ] - - operations = [ - ] diff --git a/onadata/apps/logger/migrations/0032_project_deleted_at.py b/onadata/apps/logger/migrations/0032_project_deleted_at.py deleted file mode 100644 index 2fff7faf34..0000000000 --- a/onadata/apps/logger/migrations/0032_project_deleted_at.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2017-03-20 12:27 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0031_merge'), - ] - - operations = [ - migrations.AddField( - model_name='project', - name='deleted_at', - field=models.DateTimeField(blank=True, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py deleted file mode 100644 index 3cc2db0cb1..0000000000 --- a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2017-07-05 05:59 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0032_project_deleted_at'), - ] - - operations = [ - migrations.AlterModelOptions( - name='attachment', - options={'ordering': ('pk',)}, - ), - migrations.AlterModelOptions( - name='xform', - options={ - 'ordering': ('pk',), - 'permissions': - (('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms'}, - ), - ] diff --git a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py deleted file mode 100644 index 89841453ed..0000000000 --- a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-08-14 08:32 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0033_auto_20170705_0159'), - ] - - operations = [ - migrations.AddField( - model_name='instance', - name='media_all_received', - field=models.NullBooleanField( - null=True, - verbose_name='Received All Media Attachemts' - ), - ), - migrations.AddField( - model_name='instance', - name='media_count', - field=models.PositiveIntegerField( - null=True, - verbose_name='Received Media Attachments' - ), - ), - migrations.AddField( - model_name='instance', - name='total_media', - field=models.PositiveIntegerField( - null=True, - verbose_name='Total Media Attachments' - ), - ), - ] diff --git a/onadata/apps/logger/migrations/0034_mergedxform.py b/onadata/apps/logger/migrations/0034_mergedxform.py deleted file mode 100644 index 7d68dc8566..0000000000 --- a/onadata/apps/logger/migrations/0034_mergedxform.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-07-11 14:21 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [('logger', '0033_auto_20170705_0159'), ] - - operations = [ - migrations.CreateModel( - name='MergedXForm', - fields=[ - ('xform_ptr', models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to='logger.XForm')), - ('xforms', models.ManyToManyField( - related_name='mergedxform_ptr', to='logger.XForm')), - ], - bases=('logger.xform', ), ), - ] diff --git a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py deleted file mode 100644 index 6a4578e620..0000000000 --- a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-07-12 09:29 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [('logger', '0034_mergedxform'), ] - - operations = [ - migrations.AlterModelOptions( - name='mergedxform', - options={ - 'permissions': (('view_mergedxform', 'Can view associated data' - ), ) - }, ), - ] diff --git a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py deleted file mode 100644 index 4705454762..0000000000 --- a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-07-24 11:52 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0035_auto_20170712_0529'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='is_merged_dataset', - field=models.BooleanField(default=False), - ), - ] diff --git a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py deleted file mode 100644 index e6d9231e4c..0000000000 --- a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-08-25 06:38 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0034_auto_20170814_0432'), - ('logger', '0036_xform_is_merged_dataset'), - ] - - operations = [ - ] diff --git a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py deleted file mode 100644 index 3ecb6b1668..0000000000 --- a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-08-28 21:18 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0037_merge_20170825_0238'), - ] - - operations = [ - migrations.AlterField( - model_name='instance', - name='media_all_received', - field=models.NullBooleanField( - default=True, verbose_name='Received All Media Attachemts'), ), - migrations.AlterField( - model_name='instance', - name='media_count', - field=models.PositiveIntegerField( - default=0, - null=True, - verbose_name='Received Media Attachments'), ), - migrations.AlterField( - model_name='instance', - name='total_media', - field=models.PositiveIntegerField( - default=0, null=True, verbose_name='Total Media Attachments'), - ), - ] diff --git a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py deleted file mode 100644 index 8ba7bffabb..0000000000 --- a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-09-10 00:52 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0038_auto_20170828_1718'), - ] - - operations = [ - migrations.AddField( - model_name='instance', - name='checksum', - field=models.CharField(blank=True, max_length=64, null=True), - ), - migrations.AddField( - model_name='instancehistory', - name='checksum', - field=models.CharField(blank=True, max_length=64, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py deleted file mode 100644 index 22c6225315..0000000000 --- a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-09-12 19:04 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0039_auto_20170909_2052'), - ] - - operations = [ - migrations.AlterField( - model_name='instance', - name='checksum', - field=models.CharField( - blank=True, db_index=True, max_length=64, null=True), ), - ] diff --git a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py deleted file mode 100644 index fb082d79d0..0000000000 --- a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-09-12 19:12 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0040_auto_20170912_1504'), - ] - - operations = [ - migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=249), - ), - ] diff --git a/onadata/apps/logger/migrations/0042_xform_hash.py b/onadata/apps/logger/migrations/0042_xform_hash.py deleted file mode 100644 index 8814600606..0000000000 --- a/onadata/apps/logger/migrations/0042_xform_hash.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-09-22 13:05 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0041_auto_20170912_1512'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=32, - null=True, verbose_name='Hash'), - ), - ] diff --git a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py deleted file mode 100644 index a1da0b61ad..0000000000 --- a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-10-10 08:03 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0042_xform_hash'), - ] - - operations = [ - migrations.AlterField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=36, - null=True, verbose_name='Hash'), - ), - ] diff --git a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py deleted file mode 100644 index 079550fd67..0000000000 --- a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0043_auto_20171010_0403'), - ] - - operations = [ - migrations.RunSQL( - "UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", # noqa - migrations.RunSQL.noop - ), - ] diff --git a/onadata/apps/logger/migrations/0045_attachment_name.py b/onadata/apps/logger/migrations/0045_attachment_name.py deleted file mode 100644 index 33fcc5858a..0000000000 --- a/onadata/apps/logger/migrations/0045_attachment_name.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2018-01-20 23:21 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0044_xform_hash_sql_update'), - ] - - operations = [ - migrations.AddField( - model_name='attachment', - name='name', - field=models.CharField(blank=True, max_length=100, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py deleted file mode 100644 index 982674e63a..0000000000 --- a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2018-03-14 20:18 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0045_attachment_name'), - ] - - operations = [ - migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(default='', max_length=36), - ), - ] diff --git a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py deleted file mode 100644 index 5342aa5bf8..0000000000 --- a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.11 on 2018-03-26 12:58 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0046_auto_20180314_1618'), - ] - - operations = [ - migrations.AddField( - model_name='dataview', - name='deleted_at', - field=models.DateTimeField(blank=True, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py deleted file mode 100644 index 5822b374f8..0000000000 --- a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.11 on 2018-03-28 07:25 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0047_dataview_deleted_at'), - ] - - operations = [ - migrations.AddField( - model_name='dataview', - name='deleted_by', - field=models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='dataview_deleted_by', - to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/onadata/apps/logger/migrations/0049_xform_deleted_by.py b/onadata/apps/logger/migrations/0049_xform_deleted_by.py deleted file mode 100644 index 3c862e2118..0000000000 --- a/onadata/apps/logger/migrations/0049_xform_deleted_by.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.11 on 2018-03-28 07:48 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0048_dataview_deleted_by'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='deleted_by', - field=models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/onadata/apps/logger/migrations/0050_project_deleted_by.py b/onadata/apps/logger/migrations/0050_project_deleted_by.py deleted file mode 100644 index e96a4d29dc..0000000000 --- a/onadata/apps/logger/migrations/0050_project_deleted_by.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.11 on 2018-04-26 10:12 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0049_xform_deleted_by'), - ] - - operations = [ - migrations.AddField( - model_name='project', - name='deleted_by', - field=models.ForeignKey( - blank=True, default=None, null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='project_deleted_by', - to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py deleted file mode 100644 index 24b4520e26..0000000000 --- a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Migration to re-calculate all XForm hashes. -""" -# Generated by Django 1.11.11 on 2018-05-22 15:18 -from __future__ import unicode_literals - -from django.db import migrations -from hashlib import md5 - -from onadata.libs.utils.model_tools import queryset_iterator - - -def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 - """ - Recalculate all XForm hashes. - """ - XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 - xforms = XForm.objects.filter(downloadable=True, - deleted_at__isnull=True).only('xml') - count = xforms.count() - counter = 0 - - for xform in queryset_iterator(xforms, 500): - xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest() - xform.save(update_fields=['hash']) - counter += 1 - if counter % 500 == 0: - print("Processed %d of %d forms." % (counter, count)) - - print("Processed %dforms." % counter) - - -class Migration(migrations.Migration): - """ - Migration class. - """ - - dependencies = [ - ('logger', '0050_project_deleted_by'), - ] - - operations = [ - migrations.RunPython(recalculate_xform_hash) - ] diff --git a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py deleted file mode 100644 index 364a048101..0000000000 --- a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- pylint: disable=invalid-name -# Generated by Django 1.11.13 on 2018-08-06 02:33 -"""Adds the deleted_by field to submissions. -""" -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - """Add deleted_by migration class.""" - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0051_auto_20180522_1118'), - ] - - operations = [ - migrations.AddField( - model_name='instance', - name='deleted_by', - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name='deleted_instances', - to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/onadata/apps/logger/migrations/0053_submissionreview.py b/onadata/apps/logger/migrations/0053_submissionreview.py deleted file mode 100644 index 555127c75b..0000000000 --- a/onadata/apps/logger/migrations/0053_submissionreview.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.13 on 2018-08-22 07:17 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0052_auto_20180805_2233'), - ] - - operations = [ - migrations.CreateModel( - name='SubmissionReview', - fields=[ - ('id', - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name='ID')), - ('status', - models.CharField( - choices=[('1', 'Approved'), ('3', 'Pending'), - ('2', 'Rejected')], - db_index=True, - default='3', - max_length=1, - verbose_name='Status')), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL)), - ('deleted_by', - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='deleted_reviews', - to=settings.AUTH_USER_MODEL)), - ('instance', - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='reviews', - to='logger.Instance')), - ('note', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='notes', - to='logger.Note')), - ], - ), - ] diff --git a/onadata/apps/logger/migrations/0054_instance_has_a_review.py b/onadata/apps/logger/migrations/0054_instance_has_a_review.py deleted file mode 100644 index 739b52dff0..0000000000 --- a/onadata/apps/logger/migrations/0054_instance_has_a_review.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.13 on 2018-08-30 05:28 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0053_submissionreview'), - ] - - operations = [ - migrations.AddField( - model_name='instance', - name='has_a_review', - field=models.BooleanField(default=False, verbose_name='has_a_review'), - ), - ] diff --git a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py deleted file mode 100644 index 19d80a1dbd..0000000000 --- a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.13 on 2018-09-04 11:13 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0054_instance_has_a_review'), - ] - - operations = [ - migrations.AlterField( - model_name='submissionreview', - name='deleted_at', - field=models.DateTimeField(db_index=True, default=None, null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py deleted file mode 100644 index 2b2b51e7f9..0000000000 --- a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py +++ /dev/null @@ -1,87 +0,0 @@ -# Generated by Django 2.1.5 on 2019-01-25 10:17 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import taggit.managers - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0055_auto_20180904_0713'), - ] - - operations = [ - migrations.AlterModelOptions( - name='mergedxform', - options={}, - ), - migrations.AlterModelOptions( - name='note', - options={}, - ), - migrations.AlterModelOptions( - name='project', - options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, - ), - migrations.AlterModelOptions( - name='xform', - options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, - ), - migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(blank=True, default='', max_length=100), - ), - migrations.AlterField( - model_name='instance', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.SurveyType'), - ), - migrations.AlterField( - model_name='instance', - name='user', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='osmdata', - name='field_name', - field=models.CharField(blank=True, default='', max_length=255), - ), - migrations.AlterField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default='way', max_length=10), - ), - migrations.AlterField( - model_name='project', - name='tags', - field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), - ), - migrations.AlterField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), - ), - migrations.AlterField( - model_name='widget', - name='widget_type', - field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), - ), - migrations.AlterField( - model_name='xform', - name='created_by', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='xform', - name='sms_id_string', - field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), - ), - ] diff --git a/onadata/apps/logger/migrations/0057_xform_public_key.py b/onadata/apps/logger/migrations/0057_xform_public_key.py deleted file mode 100644 index 8d1e12848b..0000000000 --- a/onadata/apps/logger/migrations/0057_xform_public_key.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.7 on 2019-10-30 13:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0056_auto_20190125_0517'), - ] - - operations = [ - migrations.AddField( - model_name='xform', - name='public_key', - field=models.TextField(default=''), - ), - ] diff --git a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py deleted file mode 100644 index 778a00a2e7..0000000000 --- a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.1 on 2019-12-11 14:00 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0057_xform_public_key'), - ] - - operations = [ - migrations.AlterField( - model_name='xform', - name='public_key', - field=models.TextField(blank=True, default='', null=True), - ), - ] diff --git a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py deleted file mode 100644 index 4a0cd2ac86..0000000000 --- a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.1.7 on 2020-02-12 08:35 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0058_auto_20191211_0900'), - ] - - operations = [ - migrations.AddField( - model_name='attachment', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py deleted file mode 100644 index 4efeb9f57a..0000000000 --- a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.9 on 2020-03-05 08:57 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0059_attachment_deleted_by'), - ] - - operations = [ - migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=36), - ), - ] diff --git a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py deleted file mode 100644 index 239cc02c2b..0000000000 --- a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py +++ /dev/null @@ -1,26 +0,0 @@ -# pylint: skip-file -# Generated by Django 2.2.10 on 2020-07-13 12:14 - -from django.db import migrations -from onadata.libs.utils.common_tools import get_uuid - - -def generate_uuid_if_missing(apps, schema_editor): - """ - Generate uuids for XForms without them - """ - XForm = apps.get_model('logger', 'XForm') - - for xform in XForm.objects.filter(uuid=''): - xform.uuid = get_uuid() - xform.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0060_auto_20200305_0357'), - ] - - operations = [ - migrations.RunPython(generate_uuid_if_missing)] diff --git a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py deleted file mode 100644 index f545fdd69a..0000000000 --- a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py +++ /dev/null @@ -1,28 +0,0 @@ -# pylint: skip-file -# Generated by Django 2.2.16 on 2021-02-02 07:48 - -from django.db import migrations -from onadata.apps.logger.models.instance import Instance - - -def regenerate_instance_json(apps, schema_editor): - """ - Regenerate Instance JSON - """ - for inst in Instance.objects.filter( - deleted_at__isnull=True, - xform__downloadable=True, - xform__deleted_at__isnull=True): - inst.json = inst.get_full_dict(load_existing=False) - inst.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0061_auto_20200713_0814'), - ] - - operations = [ - migrations.RunPython(regenerate_instance_json) - ] diff --git a/onadata/apps/logger/migrations/0063_xformversion.py b/onadata/apps/logger/migrations/0063_xformversion.py deleted file mode 100644 index 064469a5d3..0000000000 --- a/onadata/apps/logger/migrations/0063_xformversion.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 2.2.16 on 2021-03-01 14:01 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0062_auto_20210202_0248'), - ] - - operations = [ - migrations.CreateModel( - name='XFormVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xls', models.FileField(upload_to='')), - ('version', models.CharField(max_length=100)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xml', models.TextField()), - ('json', models.TextField()), - ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.XForm')), - ], - options={ - 'unique_together': {('xform', 'version')}, - }, - ), - ] diff --git a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py deleted file mode 100644 index 94b712f87e..0000000000 --- a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 2.2.16 on 2021-03-04 08:14 - -from django.db import migrations - -from onadata.apps.logger.models import XForm -from onadata.libs.utils.logger_tools import create_xform_version - - -def create_initial_xform_version(apps, schema_editor): - """ - Creates an XFormVersion object for an XForm that has no - Version - """ - queryset = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True - ) - for xform in queryset.iterator(): - if xform.version: - create_xform_version(xform, xform.user) - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0063_xformversion'), - ] - - operations = [ - migrations.RunPython(create_initial_xform_version) - ] From 0ff25ea24e5aa5b690de55981691f5e60f2a37a5 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:40:51 +0300 Subject: [PATCH 027/234] Update migrations files to depend on the squashed migration - Generate migration file for the logger application to switch the JSON Field type --- onadata/apps/api/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20220425_0340.py | 59 +++++++++++++++++++ onadata/apps/main/migrations/0001_initial.py | 2 +- .../restservice/migrations/0001_initial.py | 2 +- .../apps/viewer/migrations/0001_initial.py | 2 +- 5 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 onadata/apps/logger/migrations/0002_auto_20220425_0340.py diff --git a/onadata/apps/api/migrations/0001_initial.py b/onadata/apps/api/migrations/0001_initial.py index f4118c46a3..87e4fe8574 100644 --- a/onadata/apps/api/migrations/0001_initial.py +++ b/onadata/apps/api/migrations/0001_initial.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ('logger', '0001_pre-django-3-upgrade'), ('main', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('auth', '0001_initial'), diff --git a/onadata/apps/logger/migrations/0002_auto_20220425_0340.py b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py new file mode 100644 index 0000000000..0762d5dffe --- /dev/null +++ b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py @@ -0,0 +1,59 @@ +# Generated by Django 3.2.13 on 2022-04-25 07:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0001_pre-django-3-upgrade'), + ] + + operations = [ + migrations.AlterField( + model_name='dataview', + name='columns', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='dataview', + name='query', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='instance', + name='json', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='instance', + name='xform', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform'), + ), + migrations.AlterField( + model_name='osmdata', + name='tags', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='project', + name='metadata', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='widget', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='xform', + name='json', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='xform', + name='last_updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/onadata/apps/main/migrations/0001_initial.py b/onadata/apps/main/migrations/0001_initial.py index 6ebb6d0d76..77b160d222 100644 --- a/onadata/apps/main/migrations/0001_initial.py +++ b/onadata/apps/main/migrations/0001_initial.py @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ('logger', '0001_pre-django-3-upgrade'), ('auth', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] diff --git a/onadata/apps/restservice/migrations/0001_initial.py b/onadata/apps/restservice/migrations/0001_initial.py index f157923907..b91dd6fb0b 100644 --- a/onadata/apps/restservice/migrations/0001_initial.py +++ b/onadata/apps/restservice/migrations/0001_initial.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ('logger', '0001_pre-django-3-upgrade'), ] operations = [ diff --git a/onadata/apps/viewer/migrations/0001_initial.py b/onadata/apps/viewer/migrations/0001_initial.py index 281e34e747..822d11c437 100644 --- a/onadata/apps/viewer/migrations/0001_initial.py +++ b/onadata/apps/viewer/migrations/0001_initial.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ('logger', '0001_pre-django-3-upgrade'), ] operations = [ From 5430b462f281d8bd284755ceaadac03b07241e7a Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 10:56:47 +0300 Subject: [PATCH 028/234] Restore old logger application migration files The migration files are still necessary for old deployments and would need to be deleted after we've confirmed that all old deployments have run the new migration to keep the Migration history clean and consistent - Update migration dependencies to point back to logger 0001_initial --- onadata/apps/api/migrations/0001_initial.py | 2 +- .../apps/logger/migrations/0001_initial.py | 339 ++++++++++++++++++ .../migrations/0002_auto_20150717_0048.py | 32 ++ .../0003_dataview_instances_with_geopoints.py | 20 ++ .../migrations/0004_auto_20150910_0056.py | 59 +++ .../migrations/0005_auto_20151015_0758.py | 59 +++ .../migrations/0006_auto_20151106_0130.py | 53 +++ .../migrations/0007_osmdata_field_name.py | 20 ++ .../migrations/0008_osmdata_osm_type.py | 20 ++ .../migrations/0009_auto_20151111_0438.py | 18 + .../migrations/0010_attachment_file_size.py | 20 ++ .../0011_dataview_matches_parent.py | 20 ++ .../migrations/0012_auto_20160114_0708.py | 20 ++ .../logger/migrations/0013_note_created_by.py | 23 ++ .../migrations/0014_note_instance_field.py | 20 ++ .../migrations/0015_auto_20160222_0559.py | 25 ++ .../migrations/0016_widget_aggregation.py | 21 ++ .../migrations/0017_auto_20160224_0130.py | 32 ++ .../migrations/0018_auto_20160301_0330.py | 32 ++ .../migrations/0019_auto_20160307_0256.py | 35 ++ .../migrations/0020_auto_20160408_0325.py | 48 +++ .../migrations/0021_auto_20160408_0919.py | 21 ++ .../migrations/0022_auto_20160418_0518.py | 25 ++ .../migrations/0023_auto_20160419_0403.py | 26 ++ .../migrations/0024_xform_has_hxl_support.py | 20 ++ .../migrations/0025_xform_last_updated_at.py | 26 ++ .../migrations/0026_auto_20160913_0239.py | 20 ++ .../migrations/0027_auto_20161201_0730.py | 21 ++ .../migrations/0028_auto_20170217_0502.py | 52 +++ .../migrations/0028_auto_20170221_0838.py | 34 ++ .../migrations/0029_auto_20170221_0908.py | 22 ++ .../migrations/0030_auto_20170227_0137.py | 23 ++ onadata/apps/logger/migrations/0031_merge.py | 16 + .../migrations/0032_project_deleted_at.py | 20 ++ .../migrations/0033_auto_20170705_0159.py | 35 ++ .../migrations/0034_auto_20170814_0432.py | 39 ++ .../logger/migrations/0034_mergedxform.py | 28 ++ .../migrations/0035_auto_20170712_0529.py | 19 + .../0036_xform_is_merged_dataset.py | 20 ++ .../migrations/0037_merge_20170825_0238.py | 16 + .../migrations/0038_auto_20170828_1718.py | 33 ++ .../migrations/0039_auto_20170909_2052.py | 25 ++ .../migrations/0040_auto_20170912_1504.py | 20 ++ .../migrations/0041_auto_20170912_1512.py | 20 ++ .../apps/logger/migrations/0042_xform_hash.py | 21 ++ .../migrations/0043_auto_20171010_0403.py | 21 ++ .../migrations/0044_xform_hash_sql_update.py | 18 + .../logger/migrations/0045_attachment_name.py | 20 ++ .../migrations/0046_auto_20180314_1618.py | 20 ++ .../migrations/0047_dataview_deleted_at.py | 20 ++ .../migrations/0048_dataview_deleted_by.py | 29 ++ .../migrations/0049_xform_deleted_by.py | 28 ++ .../migrations/0050_project_deleted_by.py | 27 ++ .../migrations/0051_auto_20180522_1118.py | 45 +++ .../migrations/0052_auto_20180805_2233.py | 30 ++ .../migrations/0053_submissionreview.py | 66 ++++ .../migrations/0054_instance_has_a_review.py | 20 ++ .../migrations/0055_auto_20180904_0713.py | 20 ++ .../migrations/0056_auto_20190125_0517.py | 87 +++++ .../migrations/0057_xform_public_key.py | 18 + .../migrations/0058_auto_20191211_0900.py | 18 + .../migrations/0059_attachment_deleted_by.py | 21 ++ .../migrations/0060_auto_20200305_0357.py | 18 + .../migrations/0061_auto_20200713_0814.py | 26 ++ .../migrations/0062_auto_20210202_0248.py | 28 ++ .../logger/migrations/0063_xformversion.py | 33 ++ .../migrations/0064_auto_20210304_0314.py | 31 ++ .../restservice/migrations/0001_initial.py | 2 +- .../apps/viewer/migrations/0001_initial.py | 2 +- 69 files changed, 2165 insertions(+), 3 deletions(-) create mode 100644 onadata/apps/logger/migrations/0001_initial.py create mode 100644 onadata/apps/logger/migrations/0002_auto_20150717_0048.py create mode 100644 onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py create mode 100644 onadata/apps/logger/migrations/0004_auto_20150910_0056.py create mode 100644 onadata/apps/logger/migrations/0005_auto_20151015_0758.py create mode 100644 onadata/apps/logger/migrations/0006_auto_20151106_0130.py create mode 100644 onadata/apps/logger/migrations/0007_osmdata_field_name.py create mode 100644 onadata/apps/logger/migrations/0008_osmdata_osm_type.py create mode 100644 onadata/apps/logger/migrations/0009_auto_20151111_0438.py create mode 100644 onadata/apps/logger/migrations/0010_attachment_file_size.py create mode 100644 onadata/apps/logger/migrations/0011_dataview_matches_parent.py create mode 100644 onadata/apps/logger/migrations/0012_auto_20160114_0708.py create mode 100644 onadata/apps/logger/migrations/0013_note_created_by.py create mode 100644 onadata/apps/logger/migrations/0014_note_instance_field.py create mode 100644 onadata/apps/logger/migrations/0015_auto_20160222_0559.py create mode 100644 onadata/apps/logger/migrations/0016_widget_aggregation.py create mode 100644 onadata/apps/logger/migrations/0017_auto_20160224_0130.py create mode 100644 onadata/apps/logger/migrations/0018_auto_20160301_0330.py create mode 100644 onadata/apps/logger/migrations/0019_auto_20160307_0256.py create mode 100644 onadata/apps/logger/migrations/0020_auto_20160408_0325.py create mode 100644 onadata/apps/logger/migrations/0021_auto_20160408_0919.py create mode 100644 onadata/apps/logger/migrations/0022_auto_20160418_0518.py create mode 100644 onadata/apps/logger/migrations/0023_auto_20160419_0403.py create mode 100644 onadata/apps/logger/migrations/0024_xform_has_hxl_support.py create mode 100644 onadata/apps/logger/migrations/0025_xform_last_updated_at.py create mode 100644 onadata/apps/logger/migrations/0026_auto_20160913_0239.py create mode 100644 onadata/apps/logger/migrations/0027_auto_20161201_0730.py create mode 100644 onadata/apps/logger/migrations/0028_auto_20170217_0502.py create mode 100644 onadata/apps/logger/migrations/0028_auto_20170221_0838.py create mode 100644 onadata/apps/logger/migrations/0029_auto_20170221_0908.py create mode 100644 onadata/apps/logger/migrations/0030_auto_20170227_0137.py create mode 100644 onadata/apps/logger/migrations/0031_merge.py create mode 100644 onadata/apps/logger/migrations/0032_project_deleted_at.py create mode 100644 onadata/apps/logger/migrations/0033_auto_20170705_0159.py create mode 100644 onadata/apps/logger/migrations/0034_auto_20170814_0432.py create mode 100644 onadata/apps/logger/migrations/0034_mergedxform.py create mode 100644 onadata/apps/logger/migrations/0035_auto_20170712_0529.py create mode 100644 onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py create mode 100644 onadata/apps/logger/migrations/0037_merge_20170825_0238.py create mode 100644 onadata/apps/logger/migrations/0038_auto_20170828_1718.py create mode 100644 onadata/apps/logger/migrations/0039_auto_20170909_2052.py create mode 100644 onadata/apps/logger/migrations/0040_auto_20170912_1504.py create mode 100644 onadata/apps/logger/migrations/0041_auto_20170912_1512.py create mode 100644 onadata/apps/logger/migrations/0042_xform_hash.py create mode 100644 onadata/apps/logger/migrations/0043_auto_20171010_0403.py create mode 100644 onadata/apps/logger/migrations/0044_xform_hash_sql_update.py create mode 100644 onadata/apps/logger/migrations/0045_attachment_name.py create mode 100644 onadata/apps/logger/migrations/0046_auto_20180314_1618.py create mode 100644 onadata/apps/logger/migrations/0047_dataview_deleted_at.py create mode 100644 onadata/apps/logger/migrations/0048_dataview_deleted_by.py create mode 100644 onadata/apps/logger/migrations/0049_xform_deleted_by.py create mode 100644 onadata/apps/logger/migrations/0050_project_deleted_by.py create mode 100644 onadata/apps/logger/migrations/0051_auto_20180522_1118.py create mode 100644 onadata/apps/logger/migrations/0052_auto_20180805_2233.py create mode 100644 onadata/apps/logger/migrations/0053_submissionreview.py create mode 100644 onadata/apps/logger/migrations/0054_instance_has_a_review.py create mode 100644 onadata/apps/logger/migrations/0055_auto_20180904_0713.py create mode 100644 onadata/apps/logger/migrations/0056_auto_20190125_0517.py create mode 100644 onadata/apps/logger/migrations/0057_xform_public_key.py create mode 100644 onadata/apps/logger/migrations/0058_auto_20191211_0900.py create mode 100644 onadata/apps/logger/migrations/0059_attachment_deleted_by.py create mode 100644 onadata/apps/logger/migrations/0060_auto_20200305_0357.py create mode 100644 onadata/apps/logger/migrations/0061_auto_20200713_0814.py create mode 100644 onadata/apps/logger/migrations/0062_auto_20210202_0248.py create mode 100644 onadata/apps/logger/migrations/0063_xformversion.py create mode 100644 onadata/apps/logger/migrations/0064_auto_20210304_0314.py diff --git a/onadata/apps/api/migrations/0001_initial.py b/onadata/apps/api/migrations/0001_initial.py index 87e4fe8574..f4118c46a3 100644 --- a/onadata/apps/api/migrations/0001_initial.py +++ b/onadata/apps/api/migrations/0001_initial.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), + ('logger', '0001_initial'), ('main', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('auth', '0001_initial'), diff --git a/onadata/apps/logger/migrations/0001_initial.py b/onadata/apps/logger/migrations/0001_initial.py new file mode 100644 index 0000000000..e1d3f4fbc8 --- /dev/null +++ b/onadata/apps/logger/migrations/0001_initial.py @@ -0,0 +1,339 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import jsonfield.fields +import onadata.apps.logger.models.xform +import django.contrib.gis.db.models.fields +from django.conf import settings +import onadata.apps.logger.models.attachment +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contenttypes', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('media_file', models.FileField( + max_length=255, + upload_to=onadata.apps.logger.models.attachment.upload_to) + ), + ('mimetype', models.CharField( + default=b'', max_length=50, blank=True)), + ('extension', + models.CharField(default='non', max_length=10, db_index=True) + ), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='DataView', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('name', models.CharField(max_length=255)), + ('columns', jsonfield.fields.JSONField()), + ('query', jsonfield.fields.JSONField(default={}, blank=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ], + options={ + 'verbose_name': 'Data View', + 'verbose_name_plural': 'Data Views', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Instance', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('json', jsonfield.fields.JSONField(default={})), + ('xml', models.TextField()), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('status', models.CharField(default='submitted_via_web', + max_length=20)), + ('uuid', models.CharField(default='', max_length=249)), + ('version', models.CharField(max_length=255, null=True)), + ('geom', + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326, null=True)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='InstanceHistory', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('xml', models.TextField()), + ('uuid', models.CharField(default='', max_length=249)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('xform_instance', + models.ForeignKey(related_name='submission_history', + to='logger.Instance', + on_delete=models.CASCADE)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Note', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('note', models.TextField()), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('instance', + models.ForeignKey( + related_name='notes', to='logger.Instance', + on_delete=models.CASCADE) + ), + ], + options={ + 'permissions': (('view_note', 'View note'),), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('name', models.CharField(max_length=255)), + ('metadata', jsonfield.fields.JSONField(blank=True)), + ('shared', models.BooleanField(default=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('created_by', + models.ForeignKey(related_name='project_owner', + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE)), + ('organization', models.ForeignKey( + related_name='project_org', to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE)), + ('tags', + taggit.managers.TaggableManager( + to='taggit.Tag', through='taggit.TaggedItem', + help_text='A comma-separated list of tags.', + verbose_name='Tags')), + ('user_stars', + models.ManyToManyField(related_name='project_stars', + to=settings.AUTH_USER_MODEL)), + ], + options={ + 'permissions': ( + ('view_project', 'Can view project'), + ('add_project_xform', 'Can add xform to project'), + ('report_project_xform', + 'Can make submissions to the project'), + ('transfer_project', + 'Can transfer project to different owner'), + ('can_export_project_data', 'Can export data in project')), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='SurveyType', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('slug', models.CharField(unique=True, max_length=100)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Widget', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('object_id', models.PositiveIntegerField()), + ('widget_type', + models.CharField(default=b'charts', max_length=25, + choices=[(b'charts', b'Charts')])), + ('view_type', models.CharField(max_length=50)), + ('column', models.CharField(max_length=50)), + ('group_by', + models.CharField(default=None, max_length=50, null=True, + blank=True)), + ('title', + models.CharField(default=None, max_length=50, null=True, + blank=True)), + ('description', + models.CharField(default=None, max_length=255, + null=True, blank=True)), + ('key', + models.CharField(unique=True, max_length=32, db_index=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('content_type', + models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='XForm', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, auto_created=True, + primary_key=True)), + ('xls', + models.FileField( + null=True, + upload_to=onadata.apps.logger.models.xform.upload_to)), + ('json', models.TextField(default='')), + ('description', models.TextField(default='', null=True, + blank=True)), + ('xml', models.TextField()), + ('require_auth', models.BooleanField(default=False)), + ('shared', models.BooleanField(default=False)), + ('shared_data', models.BooleanField(default=False)), + ('downloadable', models.BooleanField(default=True)), + ('allows_sms', models.BooleanField(default=False)), + ('encrypted', models.BooleanField(default=False)), + ('sms_id_string', + models.SlugField(default=b'', verbose_name='SMS ID', + max_length=100, editable=False)), + ('id_string', + models.SlugField(verbose_name='ID', max_length=100, + editable=False)), + ('title', models.CharField(max_length=255, editable=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(null=True, blank=True)), + ('last_submission_time', + models.DateTimeField(null=True, blank=True)), + ('has_start_time', models.BooleanField(default=False)), + ('uuid', models.CharField(default='', max_length=32)), + ('bamboo_dataset', + models.CharField(default='', max_length=60)), + ('instances_with_geopoints', + models.BooleanField(default=False)), + ('instances_with_osm', models.BooleanField(default=False)), + ('num_of_submissions', models.IntegerField(default=0)), + ('version', + models.CharField(max_length=255, null=True, blank=True)), + ('created_by', + models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, + null=True, on_delete=models.CASCADE)), + ('project', models.ForeignKey(to='logger.Project', + on_delete=models.CASCADE)), + ('tags', + taggit.managers.TaggableManager( + to='taggit.Tag', through='taggit.TaggedItem', + help_text='A comma-separated list of tags.', + verbose_name='Tags')), + ('user', + models.ForeignKey(related_name='xforms', + to=settings.AUTH_USER_MODEL, null=True, + on_delete=models.CASCADE)) + ], + options={ + 'ordering': ('id_string',), + 'verbose_name': 'XForm', + 'verbose_name_plural': 'XForms', + 'permissions': ( + ('view_xform', 'Can view associated data'), + ('report_xform', 'Can make submissions to the form'), + ('move_xform', 'Can move form between projects'), + ('transfer_xform', 'Can transfer form ownership.'), + ('can_export_xform_data', 'Can export form data')), + }, + bases=(models.Model,), + ), + migrations.AlterUniqueTogether( + name='xform', + unique_together=set( + [('user', 'id_string', 'project'), + ('user', 'sms_id_string', 'project')]), + ), + migrations.AlterUniqueTogether( + name='project', + unique_together=set([('name', 'organization')]), + ), + migrations.AddField( + model_name='instance', + name='survey_type', + field=models.ForeignKey(to='logger.SurveyType', + on_delete=models.CASCADE), + preserve_default=True, + ), + migrations.AddField( + model_name='instance', + name='tags', + field=taggit.managers.TaggableManager( + to='taggit.Tag', through='taggit.TaggedItem', + help_text='A comma-separated list of tags.', + verbose_name='Tags'), + preserve_default=True, + ), + migrations.AddField( + model_name='instance', + name='user', + field=models.ForeignKey( + related_name='instances', to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='instance', + name='xform', + field=models.ForeignKey( + related_name='instances', + to='logger.XForm', null=True, on_delete=models.CASCADE), + preserve_default=True, + ), + migrations.AddField( + model_name='dataview', + name='project', + field=models.ForeignKey(to='logger.Project', + on_delete=models.CASCADE), + preserve_default=True, + ), + migrations.AddField( + model_name='dataview', + name='xform', + field=models.ForeignKey(to='logger.XForm', + on_delete=models.CASCADE), + preserve_default=True, + ), + migrations.AddField( + model_name='attachment', + name='instance', + field=models.ForeignKey( + related_name='attachments',to='logger.Instance', + on_delete=models.CASCADE), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py new file mode 100644 index 0000000000..4ae94dca71 --- /dev/null +++ b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='date_created', + field=models.DateTimeField(auto_now_add=True, null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='attachment', + name='date_modified', + field=models.DateTimeField(auto_now=True, null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='attachment', + name='deleted_at', + field=models.DateTimeField(default=None, null=True), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py new file mode 100644 index 0000000000..af36fdd965 --- /dev/null +++ b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0002_auto_20150717_0048'), + ] + + operations = [ + migrations.AddField( + model_name='dataview', + name='instances_with_geopoints', + field=models.BooleanField(default=False), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py new file mode 100644 index 0000000000..5949179146 --- /dev/null +++ b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.conf import settings + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0003_dataview_instances_with_geopoints'), + ] + + operations = [ + migrations.CreateModel( + name='ProjectGroupObjectPermission', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('content_object', models.ForeignKey( + to='logger.Project', on_delete=models.CASCADE)), + ('group', models.ForeignKey(to='auth.Group', + on_delete=models.CASCADE)), + ('permission', models.ForeignKey(to='auth.Permission', + on_delete=models.CASCADE)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='ProjectUserObjectPermission', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('content_object', models.ForeignKey(to='logger.Project', + on_delete=models.CASCADE)), + ('permission', models.ForeignKey(to='auth.Permission', + on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.AlterUniqueTogether( + name='projectuserobjectpermission', + unique_together=set([('user', 'permission', 'content_object')]), + ), + migrations.AlterUniqueTogether( + name='projectgroupobjectpermission', + unique_together=set([('group', 'permission', 'content_object')]), + ), + ] diff --git a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py new file mode 100644 index 0000000000..db63190a51 --- /dev/null +++ b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.conf import settings + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0004_auto_20150910_0056'), + ] + + operations = [ + migrations.CreateModel( + name='XFormGroupObjectPermission', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('content_object', models.ForeignKey(to='logger.XForm', + on_delete=models.CASCADE)), + ('group', models.ForeignKey(to='auth.Group', + on_delete=models.CASCADE)), + ('permission', models.ForeignKey(to='auth.Permission', + on_delete=models.CASCADE)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='XFormUserObjectPermission', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('content_object', models.ForeignKey(to='logger.XForm', + on_delete=models.CASCADE)), + ('permission', models.ForeignKey(to='auth.Permission', + on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model,), + ), + migrations.AlterUniqueTogether( + name='xformuserobjectpermission', + unique_together=set([('user', 'permission', 'content_object')]), + ), + migrations.AlterUniqueTogether( + name='xformgroupobjectpermission', + unique_together=set([('group', 'permission', 'content_object')]), + ), + ] diff --git a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py new file mode 100644 index 0000000000..aa0f1b0137 --- /dev/null +++ b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import jsonfield.fields +import django.contrib.gis.db.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0005_auto_20151015_0758'), + ] + + operations = [ + migrations.CreateModel( + name='OsmData', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('xml', models.TextField()), + ('osm_id', models.CharField(max_length=10)), + ('tags', jsonfield.fields.JSONField(default={})), + ('geom', + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326)), + ('filename', models.CharField(max_length=255)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('instance', models.ForeignKey(related_name='osm_data', + to='logger.Instance', + on_delete=models.CASCADE)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.AlterModelOptions( + name='xform', + options={ + 'ordering': ('id_string',), 'verbose_name': 'XForm', + 'verbose_name_plural': 'XForms', + 'permissions': ( + ('view_xform', 'Can view associated data'), + ('report_xform', 'Can make submissions to the form'), + ('move_xform', 'Can move form between projects'), + ('transfer_xform', 'Can transfer form ownership.'), + ('can_export_xform_data', 'Can export form data'), + ('delete_submission', 'Can delete submissions from form') + )}, + ), + ] diff --git a/onadata/apps/logger/migrations/0007_osmdata_field_name.py b/onadata/apps/logger/migrations/0007_osmdata_field_name.py new file mode 100644 index 0000000000..7870df2c4e --- /dev/null +++ b/onadata/apps/logger/migrations/0007_osmdata_field_name.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0006_auto_20151106_0130'), + ] + + operations = [ + migrations.AddField( + model_name='osmdata', + name='field_name', + field=models.CharField(default=b'', max_length=255, blank=True), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py new file mode 100644 index 0000000000..7bde859ecf --- /dev/null +++ b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0007_osmdata_field_name'), + ] + + operations = [ + migrations.AddField( + model_name='osmdata', + name='osm_type', + field=models.CharField(default=b'way', max_length=10), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py new file mode 100644 index 0000000000..96113b0393 --- /dev/null +++ b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0008_osmdata_osm_type'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='osmdata', + unique_together=set([('instance', 'field_name')]), + ), + ] diff --git a/onadata/apps/logger/migrations/0010_attachment_file_size.py b/onadata/apps/logger/migrations/0010_attachment_file_size.py new file mode 100644 index 0000000000..63bcc1c617 --- /dev/null +++ b/onadata/apps/logger/migrations/0010_attachment_file_size.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0009_auto_20151111_0438'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='file_size', + field=models.PositiveIntegerField(default=0), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py new file mode 100644 index 0000000000..cee1b46ea5 --- /dev/null +++ b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0010_attachment_file_size'), + ] + + operations = [ + migrations.AddField( + model_name='dataview', + name='matches_parent', + field=models.BooleanField(default=False), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py new file mode 100644 index 0000000000..ccbae70aaa --- /dev/null +++ b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0011_dataview_matches_parent'), + ] + + operations = [ + migrations.AlterField( + model_name='attachment', + name='mimetype', + field=models.CharField(default=b'', max_length=100, blank=True), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0013_note_created_by.py b/onadata/apps/logger/migrations/0013_note_created_by.py new file mode 100644 index 0000000000..eae6d40b75 --- /dev/null +++ b/onadata/apps/logger/migrations/0013_note_created_by.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.conf import settings + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0012_auto_20160114_0708'), + ] + + operations = [ + migrations.AddField( + model_name='note', + name='created_by', + field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, + null=True, on_delete=models.CASCADE), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0014_note_instance_field.py b/onadata/apps/logger/migrations/0014_note_instance_field.py new file mode 100644 index 0000000000..8db30ebdae --- /dev/null +++ b/onadata/apps/logger/migrations/0014_note_instance_field.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0013_note_created_by'), + ] + + operations = [ + migrations.AddField( + model_name='note', + name='instance_field', + field=models.TextField(null=True, blank=True), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py new file mode 100644 index 0000000000..eb6cbcccd4 --- /dev/null +++ b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0014_note_instance_field'), + ] + + operations = [ + migrations.AlterModelOptions( + name='widget', + options={'ordering': ('order',)}, + ), + migrations.AddField( + model_name='widget', + name='order', + field=models.PositiveIntegerField(default=0, editable=False, + db_index=True), + preserve_default=False, + ), + ] diff --git a/onadata/apps/logger/migrations/0016_widget_aggregation.py b/onadata/apps/logger/migrations/0016_widget_aggregation.py new file mode 100644 index 0000000000..d429a246ea --- /dev/null +++ b/onadata/apps/logger/migrations/0016_widget_aggregation.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0015_auto_20160222_0559'), + ] + + operations = [ + migrations.AddField( + model_name='widget', + name='aggregation', + field=models.CharField(default=None, max_length=255, null=True, + blank=True), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py new file mode 100644 index 0000000000..f6e71f7031 --- /dev/null +++ b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0016_widget_aggregation'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='uuid', + field=models.CharField(max_length=249), + preserve_default=True, + ), + migrations.AlterField( + model_name='instance', + name='xform', + field=models.ForeignKey(related_name='instances', default=-1, + to='logger.XForm', + on_delete=models.CASCADE), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='instance', + unique_together=set([('xform', 'uuid')]), + ), + ] diff --git a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py new file mode 100644 index 0000000000..077a7dce24 --- /dev/null +++ b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.conf import settings +import django.contrib.gis.db.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0017_auto_20160224_0130'), + ] + + operations = [ + migrations.AddField( + model_name='instancehistory', + name='geom', + field=django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326, null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='instancehistory', + name='user', + field=models.ForeignKey( + to=settings.AUTH_USER_MODEL, null=True, + on_delete=models.CASCADE), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py new file mode 100644 index 0000000000..bbb5f3bdfd --- /dev/null +++ b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0018_auto_20160301_0330'), + ] + + operations = [ + migrations.AddField( + model_name='widget', + name='metadata', + field=jsonfield.fields.JSONField(default={}, blank=True), + preserve_default=True, + ), + migrations.AlterField( + model_name='instance', + name='uuid', + field=models.CharField(default='', max_length=249), + preserve_default=True, + ), + migrations.AlterField( + model_name='instance', + name='xform', + field=models.ForeignKey(related_name='instances', + to='logger.XForm', null=True, + on_delete=models.CASCADE), + preserve_default=True, + ), + ] diff --git a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py new file mode 100644 index 0000000000..9353329eae --- /dev/null +++ b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-08 07:25 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0019_auto_20160307_0256'), + ] + + operations = [ + migrations.AlterField( + model_name='dataview', + name='columns', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + migrations.AlterField( + model_name='dataview', + name='query', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, + default=dict), + ), + migrations.AlterField( + model_name='instance', + name='json', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='osmdata', + name='tags', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='project', + name='metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True), + ), + migrations.AlterField( + model_name='widget', + name='metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, + default=dict), + ), + ] diff --git a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py new file mode 100644 index 0000000000..dbe27f3f87 --- /dev/null +++ b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-08 13:19 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0020_auto_20160408_0325'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ] diff --git a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py new file mode 100644 index 0000000000..8954428f43 --- /dev/null +++ b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-18 09:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0021_auto_20160408_0919'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='last_edited', + field=models.DateTimeField(default=None, null=True), + ), + migrations.AddField( + model_name='instancehistory', + name='submission_date', + field=models.DateTimeField(default=None, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py new file mode 100644 index 0000000000..dc99e5bdb0 --- /dev/null +++ b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-19 08:03 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0022_auto_20160418_0518'), + ] + + operations = [ + migrations.AlterField( + model_name='widget', + name='column', + field=models.CharField(max_length=255), + ), + migrations.AlterField( + model_name='widget', + name='group_by', + field=models.CharField(blank=True, default=None, max_length=255, + null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py new file mode 100644 index 0000000000..64a4ad8912 --- /dev/null +++ b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-19 09:54 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0023_auto_20160419_0403'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='has_hxl_support', + field=models.BooleanField(default=False), + ), + ] diff --git a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py new file mode 100644 index 0000000000..f8000e7012 --- /dev/null +++ b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-08-18 12:43 +from __future__ import unicode_literals + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0024_xform_has_hxl_support'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='last_updated_at', + field=models.DateTimeField( + auto_now=True, + default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, + tzinfo=utc)), + preserve_default=False, + ), + ] diff --git a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py new file mode 100644 index 0000000000..d29b823400 --- /dev/null +++ b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-09-13 06:39 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0025_xform_last_updated_at'), + ] + + operations = [ + migrations.AlterField( + model_name='osmdata', + name='osm_id', + field=models.CharField(max_length=20), + ), + ] diff --git a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py new file mode 100644 index 0000000000..d253875e2c --- /dev/null +++ b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-12-01 12:30 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0026_auto_20160913_0239'), + ] + + operations = [ + migrations.AlterField( + model_name='widget', + name='title', + field=models.CharField(blank=True, default=None, max_length=255, + null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py new file mode 100644 index 0000000000..39734a6f06 --- /dev/null +++ b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-02-17 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0027_auto_20161201_0730'), + ] + + operations = [ + migrations.AlterModelOptions( + name='project', + options={'permissions': ( + ('view_project', 'Can view project'), + ('add_project_xform', 'Can add xform to project'), + ('report_project_xform', + 'Can make submissions to the project'), + ('transfer_project', + 'Can transfer project to different owner'), + ('can_export_project_data', 'Can export data in project'), + ('view_project_all', 'Can view all associated data'), + ('view_project_data', 'Can view submitted data'))}, + ), + migrations.AlterModelOptions( + name='xform', + options={ + 'ordering': ('id_string',), + 'permissions': ( + ('view_xform', 'Can view associated data'), + ('view_xform_all', 'Can view all associated data'), + ('view_xform_data', 'Can view submitted data'), + ('report_xform', 'Can make submissions to the form'), + ('move_xform', 'Can move form between projects'), + ('transfer_xform', 'Can transfer form ownership.'), + ('can_export_xform_data', 'Can export form data'), + ('delete_submission', 'Can delete submissions from form')), + 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + ), + migrations.AlterField( + model_name='instance', + name='xform', + field=models.ForeignKey( + default=328, on_delete=django.db.models.deletion.CASCADE, + related_name='instances', to='logger.XForm'), + preserve_default=False, + ), + ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py new file mode 100644 index 0000000000..07c68426c3 --- /dev/null +++ b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-02-21 13:38 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('logger', '0027_auto_20161201_0730'), + ] + + operations = [ + migrations.CreateModel( + name='OpenData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, + serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('uuid', models.CharField(default='', max_length=32)), + ('object_id', models.PositiveIntegerField(blank=True, + null=True)), + ('active', models.BooleanField(default=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('content_type', models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to='contenttypes.ContentType')), + ], + ), + ] diff --git a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py new file mode 100644 index 0000000000..681b1890e9 --- /dev/null +++ b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-02-21 14:08 +from __future__ import unicode_literals + +from django.db import migrations, models +import onadata.apps.logger.models.open_data + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0028_auto_20170221_0838'), + ] + + operations = [ + migrations.AlterField( + model_name='opendata', + name='uuid', + field=models.CharField( + default=onadata.apps.logger.models.open_data.get_uuid, + max_length=32), ), + ] diff --git a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py new file mode 100644 index 0000000000..a4e879564f --- /dev/null +++ b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-02-27 06:37 +from __future__ import unicode_literals + +from django.db import migrations, models +import onadata.libs.utils.common_tools + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0029_auto_20170221_0908'), + ] + + operations = [ + migrations.AlterField( + model_name='opendata', + name='uuid', + field=models.CharField( + default=onadata.libs.utils.common_tools.get_uuid, + max_length=32, + unique=True), ), + ] diff --git a/onadata/apps/logger/migrations/0031_merge.py b/onadata/apps/logger/migrations/0031_merge.py new file mode 100644 index 0000000000..3d24ace1f5 --- /dev/null +++ b/onadata/apps/logger/migrations/0031_merge.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-02-28 08:00 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0028_auto_20170217_0502'), + ('logger', '0030_auto_20170227_0137'), + ] + + operations = [ + ] diff --git a/onadata/apps/logger/migrations/0032_project_deleted_at.py b/onadata/apps/logger/migrations/0032_project_deleted_at.py new file mode 100644 index 0000000000..2fff7faf34 --- /dev/null +++ b/onadata/apps/logger/migrations/0032_project_deleted_at.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2017-03-20 12:27 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0031_merge'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='deleted_at', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py new file mode 100644 index 0000000000..3cc2db0cb1 --- /dev/null +++ b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-07-05 05:59 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0032_project_deleted_at'), + ] + + operations = [ + migrations.AlterModelOptions( + name='attachment', + options={'ordering': ('pk',)}, + ), + migrations.AlterModelOptions( + name='xform', + options={ + 'ordering': ('pk',), + 'permissions': + (('view_xform', 'Can view associated data'), + ('view_xform_all', 'Can view all associated data'), + ('view_xform_data', 'Can view submitted data'), + ('report_xform', 'Can make submissions to the form'), + ('move_xform', 'Can move form between projects'), + ('transfer_xform', 'Can transfer form ownership.'), + ('can_export_xform_data', 'Can export form data'), + ('delete_submission', 'Can delete submissions from form')), + 'verbose_name': 'XForm', + 'verbose_name_plural': 'XForms'}, + ), + ] diff --git a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py new file mode 100644 index 0000000000..89841453ed --- /dev/null +++ b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-08-14 08:32 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0033_auto_20170705_0159'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='media_all_received', + field=models.NullBooleanField( + null=True, + verbose_name='Received All Media Attachemts' + ), + ), + migrations.AddField( + model_name='instance', + name='media_count', + field=models.PositiveIntegerField( + null=True, + verbose_name='Received Media Attachments' + ), + ), + migrations.AddField( + model_name='instance', + name='total_media', + field=models.PositiveIntegerField( + null=True, + verbose_name='Total Media Attachments' + ), + ), + ] diff --git a/onadata/apps/logger/migrations/0034_mergedxform.py b/onadata/apps/logger/migrations/0034_mergedxform.py new file mode 100644 index 0000000000..7d68dc8566 --- /dev/null +++ b/onadata/apps/logger/migrations/0034_mergedxform.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-11 14:21 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [('logger', '0033_auto_20170705_0159'), ] + + operations = [ + migrations.CreateModel( + name='MergedXForm', + fields=[ + ('xform_ptr', models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to='logger.XForm')), + ('xforms', models.ManyToManyField( + related_name='mergedxform_ptr', to='logger.XForm')), + ], + bases=('logger.xform', ), ), + ] diff --git a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py new file mode 100644 index 0000000000..6a4578e620 --- /dev/null +++ b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-12 09:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [('logger', '0034_mergedxform'), ] + + operations = [ + migrations.AlterModelOptions( + name='mergedxform', + options={ + 'permissions': (('view_mergedxform', 'Can view associated data' + ), ) + }, ), + ] diff --git a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py new file mode 100644 index 0000000000..4705454762 --- /dev/null +++ b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-24 11:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0035_auto_20170712_0529'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='is_merged_dataset', + field=models.BooleanField(default=False), + ), + ] diff --git a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py new file mode 100644 index 0000000000..e6d9231e4c --- /dev/null +++ b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-08-25 06:38 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0034_auto_20170814_0432'), + ('logger', '0036_xform_is_merged_dataset'), + ] + + operations = [ + ] diff --git a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py new file mode 100644 index 0000000000..3ecb6b1668 --- /dev/null +++ b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-08-28 21:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0037_merge_20170825_0238'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='media_all_received', + field=models.NullBooleanField( + default=True, verbose_name='Received All Media Attachemts'), ), + migrations.AlterField( + model_name='instance', + name='media_count', + field=models.PositiveIntegerField( + default=0, + null=True, + verbose_name='Received Media Attachments'), ), + migrations.AlterField( + model_name='instance', + name='total_media', + field=models.PositiveIntegerField( + default=0, null=True, verbose_name='Total Media Attachments'), + ), + ] diff --git a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py new file mode 100644 index 0000000000..8ba7bffabb --- /dev/null +++ b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-09-10 00:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0038_auto_20170828_1718'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='checksum', + field=models.CharField(blank=True, max_length=64, null=True), + ), + migrations.AddField( + model_name='instancehistory', + name='checksum', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py new file mode 100644 index 0000000000..22c6225315 --- /dev/null +++ b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-09-12 19:04 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0039_auto_20170909_2052'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='checksum', + field=models.CharField( + blank=True, db_index=True, max_length=64, null=True), ), + ] diff --git a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py new file mode 100644 index 0000000000..fb082d79d0 --- /dev/null +++ b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-09-12 19:12 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0040_auto_20170912_1504'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='uuid', + field=models.CharField(db_index=True, default='', max_length=249), + ), + ] diff --git a/onadata/apps/logger/migrations/0042_xform_hash.py b/onadata/apps/logger/migrations/0042_xform_hash.py new file mode 100644 index 0000000000..8814600606 --- /dev/null +++ b/onadata/apps/logger/migrations/0042_xform_hash.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-09-22 13:05 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0041_auto_20170912_1512'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='hash', + field=models.CharField(blank=True, default=None, max_length=32, + null=True, verbose_name='Hash'), + ), + ] diff --git a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py new file mode 100644 index 0000000000..a1da0b61ad --- /dev/null +++ b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-10-10 08:03 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0042_xform_hash'), + ] + + operations = [ + migrations.AlterField( + model_name='xform', + name='hash', + field=models.CharField(blank=True, default=None, max_length=36, + null=True, verbose_name='Hash'), + ), + ] diff --git a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py new file mode 100644 index 0000000000..079550fd67 --- /dev/null +++ b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0043_auto_20171010_0403'), + ] + + operations = [ + migrations.RunSQL( + "UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", # noqa + migrations.RunSQL.noop + ), + ] diff --git a/onadata/apps/logger/migrations/0045_attachment_name.py b/onadata/apps/logger/migrations/0045_attachment_name.py new file mode 100644 index 0000000000..33fcc5858a --- /dev/null +++ b/onadata/apps/logger/migrations/0045_attachment_name.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2018-01-20 23:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0044_xform_hash_sql_update'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='name', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py new file mode 100644 index 0000000000..982674e63a --- /dev/null +++ b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2018-03-14 20:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0045_attachment_name'), + ] + + operations = [ + migrations.AlterField( + model_name='xform', + name='uuid', + field=models.CharField(default='', max_length=36), + ), + ] diff --git a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py new file mode 100644 index 0000000000..5342aa5bf8 --- /dev/null +++ b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-03-26 12:58 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0046_auto_20180314_1618'), + ] + + operations = [ + migrations.AddField( + model_name='dataview', + name='deleted_at', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py new file mode 100644 index 0000000000..5822b374f8 --- /dev/null +++ b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-03-28 07:25 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0047_dataview_deleted_at'), + ] + + operations = [ + migrations.AddField( + model_name='dataview', + name='deleted_by', + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='dataview_deleted_by', + to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/onadata/apps/logger/migrations/0049_xform_deleted_by.py b/onadata/apps/logger/migrations/0049_xform_deleted_by.py new file mode 100644 index 0000000000..3c862e2118 --- /dev/null +++ b/onadata/apps/logger/migrations/0049_xform_deleted_by.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-03-28 07:48 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0048_dataview_deleted_by'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='deleted_by', + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/onadata/apps/logger/migrations/0050_project_deleted_by.py b/onadata/apps/logger/migrations/0050_project_deleted_by.py new file mode 100644 index 0000000000..e96a4d29dc --- /dev/null +++ b/onadata/apps/logger/migrations/0050_project_deleted_by.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-04-26 10:12 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0049_xform_deleted_by'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='deleted_by', + field=models.ForeignKey( + blank=True, default=None, null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='project_deleted_by', + to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py new file mode 100644 index 0000000000..24b4520e26 --- /dev/null +++ b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +Migration to re-calculate all XForm hashes. +""" +# Generated by Django 1.11.11 on 2018-05-22 15:18 +from __future__ import unicode_literals + +from django.db import migrations +from hashlib import md5 + +from onadata.libs.utils.model_tools import queryset_iterator + + +def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 + """ + Recalculate all XForm hashes. + """ + XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, + deleted_at__isnull=True).only('xml') + count = xforms.count() + counter = 0 + + for xform in queryset_iterator(xforms, 500): + xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest() + xform.save(update_fields=['hash']) + counter += 1 + if counter % 500 == 0: + print("Processed %d of %d forms." % (counter, count)) + + print("Processed %dforms." % counter) + + +class Migration(migrations.Migration): + """ + Migration class. + """ + + dependencies = [ + ('logger', '0050_project_deleted_by'), + ] + + operations = [ + migrations.RunPython(recalculate_xform_hash) + ] diff --git a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py new file mode 100644 index 0000000000..364a048101 --- /dev/null +++ b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- pylint: disable=invalid-name +# Generated by Django 1.11.13 on 2018-08-06 02:33 +"""Adds the deleted_by field to submissions. +""" +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + """Add deleted_by migration class.""" + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0051_auto_20180522_1118'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='deleted_by', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='deleted_instances', + to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/onadata/apps/logger/migrations/0053_submissionreview.py b/onadata/apps/logger/migrations/0053_submissionreview.py new file mode 100644 index 0000000000..555127c75b --- /dev/null +++ b/onadata/apps/logger/migrations/0053_submissionreview.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-08-22 07:17 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0052_auto_20180805_2233'), + ] + + operations = [ + migrations.CreateModel( + name='SubmissionReview', + fields=[ + ('id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('status', + models.CharField( + choices=[('1', 'Approved'), ('3', 'Pending'), + ('2', 'Rejected')], + db_index=True, + default='3', + max_length=1, + verbose_name='Status')), + ('deleted_at', models.DateTimeField(default=None, null=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('created_by', + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL)), + ('deleted_by', + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='deleted_reviews', + to=settings.AUTH_USER_MODEL)), + ('instance', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='reviews', + to='logger.Instance')), + ('note', + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='notes', + to='logger.Note')), + ], + ), + ] diff --git a/onadata/apps/logger/migrations/0054_instance_has_a_review.py b/onadata/apps/logger/migrations/0054_instance_has_a_review.py new file mode 100644 index 0000000000..739b52dff0 --- /dev/null +++ b/onadata/apps/logger/migrations/0054_instance_has_a_review.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-08-30 05:28 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0053_submissionreview'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='has_a_review', + field=models.BooleanField(default=False, verbose_name='has_a_review'), + ), + ] diff --git a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py new file mode 100644 index 0000000000..19d80a1dbd --- /dev/null +++ b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-09-04 11:13 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0054_instance_has_a_review'), + ] + + operations = [ + migrations.AlterField( + model_name='submissionreview', + name='deleted_at', + field=models.DateTimeField(db_index=True, default=None, null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py new file mode 100644 index 0000000000..2b2b51e7f9 --- /dev/null +++ b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py @@ -0,0 +1,87 @@ +# Generated by Django 2.1.5 on 2019-01-25 10:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0055_auto_20180904_0713'), + ] + + operations = [ + migrations.AlterModelOptions( + name='mergedxform', + options={}, + ), + migrations.AlterModelOptions( + name='note', + options={}, + ), + migrations.AlterModelOptions( + name='project', + options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, + ), + migrations.AlterModelOptions( + name='xform', + options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + ), + migrations.AlterField( + model_name='attachment', + name='mimetype', + field=models.CharField(blank=True, default='', max_length=100), + ), + migrations.AlterField( + model_name='instance', + name='deleted_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='instance', + name='survey_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.SurveyType'), + ), + migrations.AlterField( + model_name='instance', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='osmdata', + name='field_name', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='osmdata', + name='osm_type', + field=models.CharField(default='way', max_length=10), + ), + migrations.AlterField( + model_name='project', + name='tags', + field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + ), + migrations.AlterField( + model_name='widget', + name='order', + field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + ), + migrations.AlterField( + model_name='widget', + name='widget_type', + field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), + ), + migrations.AlterField( + model_name='xform', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='xform', + name='sms_id_string', + field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), + ), + ] diff --git a/onadata/apps/logger/migrations/0057_xform_public_key.py b/onadata/apps/logger/migrations/0057_xform_public_key.py new file mode 100644 index 0000000000..8d1e12848b --- /dev/null +++ b/onadata/apps/logger/migrations/0057_xform_public_key.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.7 on 2019-10-30 13:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0056_auto_20190125_0517'), + ] + + operations = [ + migrations.AddField( + model_name='xform', + name='public_key', + field=models.TextField(default=''), + ), + ] diff --git a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py new file mode 100644 index 0000000000..778a00a2e7 --- /dev/null +++ b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.1 on 2019-12-11 14:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0057_xform_public_key'), + ] + + operations = [ + migrations.AlterField( + model_name='xform', + name='public_key', + field=models.TextField(blank=True, default='', null=True), + ), + ] diff --git a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py new file mode 100644 index 0000000000..4a0cd2ac86 --- /dev/null +++ b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py @@ -0,0 +1,21 @@ +# Generated by Django 2.1.7 on 2020-02-12 08:35 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0058_auto_20191211_0900'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='deleted_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py new file mode 100644 index 0000000000..4efeb9f57a --- /dev/null +++ b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.9 on 2020-03-05 08:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0059_attachment_deleted_by'), + ] + + operations = [ + migrations.AlterField( + model_name='xform', + name='uuid', + field=models.CharField(db_index=True, default='', max_length=36), + ), + ] diff --git a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py new file mode 100644 index 0000000000..239cc02c2b --- /dev/null +++ b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py @@ -0,0 +1,26 @@ +# pylint: skip-file +# Generated by Django 2.2.10 on 2020-07-13 12:14 + +from django.db import migrations +from onadata.libs.utils.common_tools import get_uuid + + +def generate_uuid_if_missing(apps, schema_editor): + """ + Generate uuids for XForms without them + """ + XForm = apps.get_model('logger', 'XForm') + + for xform in XForm.objects.filter(uuid=''): + xform.uuid = get_uuid() + xform.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0060_auto_20200305_0357'), + ] + + operations = [ + migrations.RunPython(generate_uuid_if_missing)] diff --git a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py new file mode 100644 index 0000000000..f545fdd69a --- /dev/null +++ b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py @@ -0,0 +1,28 @@ +# pylint: skip-file +# Generated by Django 2.2.16 on 2021-02-02 07:48 + +from django.db import migrations +from onadata.apps.logger.models.instance import Instance + + +def regenerate_instance_json(apps, schema_editor): + """ + Regenerate Instance JSON + """ + for inst in Instance.objects.filter( + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True): + inst.json = inst.get_full_dict(load_existing=False) + inst.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0061_auto_20200713_0814'), + ] + + operations = [ + migrations.RunPython(regenerate_instance_json) + ] diff --git a/onadata/apps/logger/migrations/0063_xformversion.py b/onadata/apps/logger/migrations/0063_xformversion.py new file mode 100644 index 0000000000..064469a5d3 --- /dev/null +++ b/onadata/apps/logger/migrations/0063_xformversion.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.16 on 2021-03-01 14:01 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0062_auto_20210202_0248'), + ] + + operations = [ + migrations.CreateModel( + name='XFormVersion', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xls', models.FileField(upload_to='')), + ('version', models.CharField(max_length=100)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ('xml', models.TextField()), + ('json', models.TextField()), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.XForm')), + ], + options={ + 'unique_together': {('xform', 'version')}, + }, + ), + ] diff --git a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py new file mode 100644 index 0000000000..94b712f87e --- /dev/null +++ b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.16 on 2021-03-04 08:14 + +from django.db import migrations + +from onadata.apps.logger.models import XForm +from onadata.libs.utils.logger_tools import create_xform_version + + +def create_initial_xform_version(apps, schema_editor): + """ + Creates an XFormVersion object for an XForm that has no + Version + """ + queryset = XForm.objects.filter( + downloadable=True, + deleted_at__isnull=True + ) + for xform in queryset.iterator(): + if xform.version: + create_xform_version(xform, xform.user) + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0063_xformversion'), + ] + + operations = [ + migrations.RunPython(create_initial_xform_version) + ] diff --git a/onadata/apps/restservice/migrations/0001_initial.py b/onadata/apps/restservice/migrations/0001_initial.py index b91dd6fb0b..f157923907 100644 --- a/onadata/apps/restservice/migrations/0001_initial.py +++ b/onadata/apps/restservice/migrations/0001_initial.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), + ('logger', '0001_initial'), ] operations = [ diff --git a/onadata/apps/viewer/migrations/0001_initial.py b/onadata/apps/viewer/migrations/0001_initial.py index 822d11c437..281e34e747 100644 --- a/onadata/apps/viewer/migrations/0001_initial.py +++ b/onadata/apps/viewer/migrations/0001_initial.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), + ('logger', '0001_initial'), ] operations = [ From e756f4c688646cdae6141558b421bc5d6f70848e Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 11:10:08 +0300 Subject: [PATCH 029/234] Revert dependency changes to use 0001_initial --- onadata/apps/main/migrations/0001_initial.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onadata/apps/main/migrations/0001_initial.py b/onadata/apps/main/migrations/0001_initial.py index 77b160d222..b3faf5e95a 100644 --- a/onadata/apps/main/migrations/0001_initial.py +++ b/onadata/apps/main/migrations/0001_initial.py @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), + ('logger', '0001_initial'), ('auth', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] @@ -257,3 +257,4 @@ class Migration(migrations.Migration): unique_together=set([('xform', 'data_type', 'data_value')]), ), ] + From 1d4090de03a5e3d49bef5f29f0fca2cfb9d278f2 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 11:10:35 +0300 Subject: [PATCH 030/234] Add back `replaces` attribute Avoids multiple leaf node conflicts on existing databases --- onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py index 1ea9f64186..3dee4c6a7a 100644 --- a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py +++ b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py @@ -76,6 +76,8 @@ def create_initial_xform_version(apps, schema_editor): class Migration(migrations.Migration): + replaces = [('logger', '0001_initial'), ('logger', '0002_auto_20150717_0048'), ('logger', '0003_dataview_instances_with_geopoints'), ('logger', '0004_auto_20150910_0056'), ('logger', '0005_auto_20151015_0758'), ('logger', '0006_auto_20151106_0130'), ('logger', '0007_osmdata_field_name'), ('logger', '0008_osmdata_osm_type'), ('logger', '0009_auto_20151111_0438'), ('logger', '0010_attachment_file_size'), ('logger', '0011_dataview_matches_parent'), ('logger', '0012_auto_20160114_0708'), ('logger', '0013_note_created_by'), ('logger', '0014_note_instance_field'), ('logger', '0015_auto_20160222_0559'), ('logger', '0016_widget_aggregation'), ('logger', '0017_auto_20160224_0130'), ('logger', '0018_auto_20160301_0330'), ('logger', '0019_auto_20160307_0256'), ('logger', '0020_auto_20160408_0325'), ('logger', '0021_auto_20160408_0919'), ('logger', '0022_auto_20160418_0518'), ('logger', '0023_auto_20160419_0403'), ('logger', '0024_xform_has_hxl_support'), ('logger', '0025_xform_last_updated_at'), ('logger', '0026_auto_20160913_0239'), ('logger', '0027_auto_20161201_0730'), ('logger', '0028_auto_20170221_0838'), ('logger', '0029_auto_20170221_0908'), ('logger', '0030_auto_20170227_0137'), ('logger', '0028_auto_20170217_0502'), ('logger', '0031_merge'), ('logger', '0032_project_deleted_at'), ('logger', '0033_auto_20170705_0159'), ('logger', '0034_mergedxform'), ('logger', '0035_auto_20170712_0529'), ('logger', '0036_xform_is_merged_dataset'), ('logger', '0034_auto_20170814_0432'), ('logger', '0037_merge_20170825_0238'), ('logger', '0038_auto_20170828_1718'), ('logger', '0039_auto_20170909_2052'), ('logger', '0040_auto_20170912_1504'), ('logger', '0041_auto_20170912_1512'), ('logger', '0042_xform_hash'), ('logger', '0043_auto_20171010_0403'), ('logger', '0044_xform_hash_sql_update'), ('logger', '0045_attachment_name'), ('logger', '0046_auto_20180314_1618'), ('logger', '0047_dataview_deleted_at'), ('logger', '0048_dataview_deleted_by'), ('logger', '0049_xform_deleted_by'), ('logger', '0050_project_deleted_by'), ('logger', '0051_auto_20180522_1118'), ('logger', '0052_auto_20180805_2233'), ('logger', '0053_submissionreview'), ('logger', '0054_instance_has_a_review'), ('logger', '0055_auto_20180904_0713'), ('logger', '0056_auto_20190125_0517'), ('logger', '0057_xform_public_key'), ('logger', '0058_auto_20191211_0900'), ('logger', '0059_attachment_deleted_by'), ('logger', '0060_auto_20200305_0357'), ('logger', '0061_auto_20200713_0814'), ('logger', '0062_auto_20210202_0248'), ('logger', '0063_xformversion'), ('logger', '0064_auto_20210304_0314')] + initial = True dependencies = [ From c8ee2b3ddbdff4c4775d49135808445f4615e57d Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 11:12:53 +0300 Subject: [PATCH 031/234] Squash viewer app migrations --- .../migrations/0001_pre-django-3-upgrade.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py diff --git a/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py new file mode 100644 index 0000000000..b4ce22c289 --- /dev/null +++ b/onadata/apps/viewer/migrations/0001_pre-django-3-upgrade.py @@ -0,0 +1,66 @@ +# Generated by Django 3.2.13 on 2022-04-25 08:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('viewer', '0001_initial'), ('viewer', '0002_export_options'), ('viewer', '0003_auto_20151226_0100'), ('viewer', '0004_auto_20151226_0109'), ('viewer', '0005_auto_20160408_0325'), ('viewer', '0006_auto_20160418_0525'), ('viewer', '0007_export_error_message'), ('viewer', '0008_auto_20190125_0517'), ('viewer', '0009_alter_export_options')] + + initial = True + + dependencies = [ + ('logger', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ColumnRename', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xpath', models.CharField(max_length=255, unique=True)), + ('column_name', models.CharField(max_length=32)), + ], + ), + migrations.CreateModel( + name='ParsedInstance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('start_time', models.DateTimeField(null=True)), + ('end_time', models.DateTimeField(null=True)), + ('lat', models.FloatField(null=True)), + ('lng', models.FloatField(null=True)), + ('instance', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='parsed_instance', to='logger.instance')), + ], + ), + migrations.CreateModel( + name='DataDictionary', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('logger.xform',), + ), + migrations.CreateModel( + name='Export', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_on', models.DateTimeField(auto_now_add=True)), + ('filename', models.CharField(blank=True, max_length=255, null=True)), + ('filedir', models.CharField(blank=True, max_length=255, null=True)), + ('export_type', models.CharField(choices=[('xls', 'Excel'), ('csv', 'CSV'), ('zip', 'ZIP'), ('kml', 'kml'), ('csv_zip', 'CSV ZIP'), ('sav_zip', 'SAV ZIP'), ('sav', 'SAV'), ('external', 'Excel'), ('osm', 'osm'), ('gsheets', 'Google Sheets')], default='xls', max_length=10)), + ('task_id', models.CharField(blank=True, max_length=255, null=True)), + ('time_of_last_submission', models.DateTimeField(default=None, null=True)), + ('internal_status', models.SmallIntegerField(default=0)), + ('export_url', models.URLField(default=None, null=True)), + ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), + ('options', models.JSONField(default=dict)), + ('error_message', models.CharField(blank=True, max_length=255, null=True)), + ], + options={ + 'unique_together': {('xform', 'filename')}, + }, + ), + ] From 31b75576f03efe24846bcd98b8f9acaf9a106afd Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 11:14:12 +0300 Subject: [PATCH 032/234] Install `jsonfield` package; Allows migrations to run on the CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 736a82b0e8..47c823c506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: pip install -U pip pip install -r requirements/base.pip pip install flake8 + # Install jsonfield packages necessary for the Django <3 migrations + pip install jsonfield - name: Run tests run: | From 6a0e6d9184cc079065446ddfeb8a30588056344d Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 11:27:48 +0300 Subject: [PATCH 033/234] Alter media_all_received col to use BooleanField Signed-off-by: Kipchirchir Sigei --- .../0003_alter_instance_media_all_received.py | 18 ++++++++++++++++++ onadata/apps/logger/models/instance.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py diff --git a/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py new file mode 100644 index 0000000000..195a3aecfb --- /dev/null +++ b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-25 08:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0002_auto_20220425_0340'), + ] + + operations = [ + migrations.AlterField( + model_name='instance', + name='media_all_received', + field=models.BooleanField(default=True, null=True, verbose_name='Received All Media Attachemts'), + ), + ] diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 609aa51480..e73fa09d5d 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -569,7 +569,7 @@ class Instance(models.Model, InstanceBaseClass): geom = models.GeometryCollectionField(null=True) # Keep track of whether all media attachments have been received - media_all_received = models.NullBooleanField( + media_all_received = models.BooleanField( _("Received All Media Attachemts"), null=True, default=True ) total_media = models.PositiveIntegerField( From c4f2c9f0d62fe06cbee66b044a7ca9fd27c5d6de Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 25 Apr 2022 12:08:47 +0300 Subject: [PATCH 034/234] Fix failing `query_data` test Update the test to expect an ID string instead of an integer; When the row is queried on the database it seems the returned response is now a string type instead of it being an integer --- onadata/apps/logger/tests/models/test_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 27124f34c3..99bfb6e7c7 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -180,7 +180,7 @@ def test_query_filter_by_integer(self): data = [i.get('_id') for i in query_data( self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')] self.assertEqual(len(data), 1) - self.assertEqual(data, [oldest]) + self.assertEqual(data, [str(oldest)]) # mongo $gt data = [i.get('_id') for i in query_data( From e62a8faae20a74912f4bcb9f5fe32e5e49d7bd1d Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Mon, 25 Apr 2022 12:54:06 +0300 Subject: [PATCH 035/234] Fix cursor.fetchall() returning stringofied dicts Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/data_view.py | 2 +- onadata/apps/main/models/audit.py | 2 +- onadata/apps/viewer/models/parsed_instance.py | 2 +- onadata/settings/common.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index a0983dd3dc..3e79088249 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -197,7 +197,7 @@ def has_instance(self, instance): cursor.execute(sql, [text(i) for i in params]) for row in cursor.fetchall(): - records = row[0] + records = json.loads(row[0]) return True if records else False diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index c6bf06e300..ab0a7146c3 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -51,7 +51,7 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): if fields is None: for row in cursor.fetchall(): - yield row[0] + yield json.loads(row[0]) else: for row in cursor.fetchall(): yield dict(zip(fields, row)) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 30bd1e2b4d..6d6b70f2bc 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -95,7 +95,7 @@ def _query_iterator(sql, fields=None, params=[], count=False): if fields is None: for row in cursor.fetchall(): - yield row[0] + yield json.loads(row[0]) else: for row in cursor.fetchall(): yield dict(zip(fields, row)) diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 3ade3e2e9b..665dc8ea83 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -144,6 +144,7 @@ "django.template.context_processors.media", "django.template.context_processors.static", "django.template.context_processors.tz", + "django.template.context_processors.request", "django.contrib.messages.context_processors.messages", "onadata.apps.main.context_processors.google_analytics", "onadata.apps.main.context_processors.site_name", From 238335a812ae6ad855f3bd0bb6204271f98337e6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 26 Apr 2022 06:16:07 +0300 Subject: [PATCH 036/234] remove jsonfield import from migrations --- .../apps/logger/migrations/0001_initial.py | 596 +++++++++++------- .../migrations/0006_auto_20151106_0130.py | 77 ++- .../migrations/0019_auto_20160307_0256.py | 28 +- onadata/apps/main/migrations/0001_initial.py | 588 ++++++++++------- .../viewer/migrations/0002_export_options.py | 11 +- 5 files changed, 795 insertions(+), 505 deletions(-) diff --git a/onadata/apps/logger/migrations/0001_initial.py b/onadata/apps/logger/migrations/0001_initial.py index e1d3f4fbc8..f102ed93b5 100644 --- a/onadata/apps/logger/migrations/0001_initial.py +++ b/onadata/apps/logger/migrations/0001_initial.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields import onadata.apps.logger.models.xform import django.contrib.gis.db.models.fields from django.conf import settings @@ -13,327 +12,452 @@ class Migration(migrations.Migration): dependencies = [ - ('taggit', '0001_initial'), + ("taggit", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('contenttypes', '0001_initial'), + ("contenttypes", "0001_initial"), ] operations = [ migrations.CreateModel( - name='Attachment', + name="Attachment", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('media_file', models.FileField( - max_length=255, - upload_to=onadata.apps.logger.models.attachment.upload_to) - ), - ('mimetype', models.CharField( - default=b'', max_length=50, blank=True)), - ('extension', - models.CharField(default='non', max_length=10, db_index=True) - ), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "media_file", + models.FileField( + max_length=255, + upload_to=onadata.apps.logger.models.attachment.upload_to, + ), + ), + ("mimetype", models.CharField(default=b"", max_length=50, blank=True)), + ( + "extension", + models.CharField(default="non", max_length=10, db_index=True), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='DataView', + name="DataView", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('columns', jsonfield.fields.JSONField()), - ('query', jsonfield.fields.JSONField(default={}, blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255)), + ("columns", models.JSONField()), + ("query", models.JSONField(default={}, blank=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), ], options={ - 'verbose_name': 'Data View', - 'verbose_name_plural': 'Data Views', + "verbose_name": "Data View", + "verbose_name_plural": "Data Views", }, bases=(models.Model,), ), migrations.CreateModel( - name='Instance', + name="Instance", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('json', jsonfield.fields.JSONField(default={})), - ('xml', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('status', models.CharField(default='submitted_via_web', - max_length=20)), - ('uuid', models.CharField(default='', max_length=249)), - ('version', models.CharField(max_length=255, null=True)), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("json", models.JSONField(default={})), + ("xml", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "status", + models.CharField(default="submitted_via_web", max_length=20), + ), + ("uuid", models.CharField(default="", max_length=249)), + ("version", models.CharField(max_length=255, null=True)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326, null=True + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='InstanceHistory', + name="InstanceHistory", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xml', models.TextField()), - ('uuid', models.CharField(default='', max_length=249)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xform_instance', - models.ForeignKey(related_name='submission_history', - to='logger.Instance', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("xml", models.TextField()), + ("uuid", models.CharField(default="", max_length=249)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "xform_instance", + models.ForeignKey( + related_name="submission_history", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='Note', + name="Note", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('note', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('instance', - models.ForeignKey( - related_name='notes', to='logger.Instance', - on_delete=models.CASCADE) - ), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("note", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "instance", + models.ForeignKey( + related_name="notes", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], options={ - 'permissions': (('view_note', 'View note'),), + "permissions": (("view_note", "View note"),), }, bases=(models.Model,), ), migrations.CreateModel( - name='Project', + name="Project", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('name', models.CharField(max_length=255)), - ('metadata', jsonfield.fields.JSONField(blank=True)), - ('shared', models.BooleanField(default=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey(related_name='project_owner', - to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('organization', models.ForeignKey( - related_name='project_org', to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user_stars', - models.ManyToManyField(related_name='project_stars', - to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255)), + ("metadata", models.JSONField(blank=True)), + ("shared", models.BooleanField(default=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + related_name="project_owner", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ( + "organization", + models.ForeignKey( + related_name="project_org", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), + ), + ( + "user_stars", + models.ManyToManyField( + related_name="project_stars", to=settings.AUTH_USER_MODEL + ), + ), ], options={ - 'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project')), + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ), }, bases=(models.Model,), ), migrations.CreateModel( - name='SurveyType', + name="SurveyType", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('slug', models.CharField(unique=True, max_length=100)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("slug", models.CharField(unique=True, max_length=100)), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='Widget', + name="Widget", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('object_id', models.PositiveIntegerField()), - ('widget_type', - models.CharField(default=b'charts', max_length=25, - choices=[(b'charts', b'Charts')])), - ('view_type', models.CharField(max_length=50)), - ('column', models.CharField(max_length=50)), - ('group_by', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('title', - models.CharField(default=None, max_length=50, null=True, - blank=True)), - ('description', - models.CharField(default=None, max_length=255, - null=True, blank=True)), - ('key', - models.CharField(unique=True, max_length=32, db_index=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', - models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("object_id", models.PositiveIntegerField()), + ( + "widget_type", + models.CharField( + default=b"charts", + max_length=25, + choices=[(b"charts", b"Charts")], + ), + ), + ("view_type", models.CharField(max_length=50)), + ("column", models.CharField(max_length=50)), + ( + "group_by", + models.CharField( + default=None, max_length=50, null=True, blank=True + ), + ), + ( + "title", + models.CharField( + default=None, max_length=50, null=True, blank=True + ), + ), + ( + "description", + models.CharField( + default=None, max_length=255, null=True, blank=True + ), + ), + ("key", models.CharField(unique=True, max_length=32, db_index=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + to="contenttypes.ContentType", on_delete=models.CASCADE + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='XForm', + name="XForm", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('xls', - models.FileField( - null=True, - upload_to=onadata.apps.logger.models.xform.upload_to)), - ('json', models.TextField(default='')), - ('description', models.TextField(default='', null=True, - blank=True)), - ('xml', models.TextField()), - ('require_auth', models.BooleanField(default=False)), - ('shared', models.BooleanField(default=False)), - ('shared_data', models.BooleanField(default=False)), - ('downloadable', models.BooleanField(default=True)), - ('allows_sms', models.BooleanField(default=False)), - ('encrypted', models.BooleanField(default=False)), - ('sms_id_string', - models.SlugField(default=b'', verbose_name='SMS ID', - max_length=100, editable=False)), - ('id_string', - models.SlugField(verbose_name='ID', max_length=100, - editable=False)), - ('title', models.CharField(max_length=255, editable=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(null=True, blank=True)), - ('last_submission_time', - models.DateTimeField(null=True, blank=True)), - ('has_start_time', models.BooleanField(default=False)), - ('uuid', models.CharField(default='', max_length=32)), - ('bamboo_dataset', - models.CharField(default='', max_length=60)), - ('instances_with_geopoints', - models.BooleanField(default=False)), - ('instances_with_osm', models.BooleanField(default=False)), - ('num_of_submissions', models.IntegerField(default=0)), - ('version', - models.CharField(max_length=255, null=True, blank=True)), - ('created_by', - models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE)), - ('project', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('tags', - taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags')), - ('user', - models.ForeignKey(related_name='xforms', - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE)) + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "xls", + models.FileField( + null=True, upload_to=onadata.apps.logger.models.xform.upload_to + ), + ), + ("json", models.TextField(default="")), + ("description", models.TextField(default="", null=True, blank=True)), + ("xml", models.TextField()), + ("require_auth", models.BooleanField(default=False)), + ("shared", models.BooleanField(default=False)), + ("shared_data", models.BooleanField(default=False)), + ("downloadable", models.BooleanField(default=True)), + ("allows_sms", models.BooleanField(default=False)), + ("encrypted", models.BooleanField(default=False)), + ( + "sms_id_string", + models.SlugField( + default=b"", + verbose_name="SMS ID", + max_length=100, + editable=False, + ), + ), + ( + "id_string", + models.SlugField(verbose_name="ID", max_length=100, editable=False), + ), + ("title", models.CharField(max_length=255, editable=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(null=True, blank=True)), + ("last_submission_time", models.DateTimeField(null=True, blank=True)), + ("has_start_time", models.BooleanField(default=False)), + ("uuid", models.CharField(default="", max_length=32)), + ("bamboo_dataset", models.CharField(default="", max_length=60)), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("instances_with_osm", models.BooleanField(default=False)), + ("num_of_submissions", models.IntegerField(default=0)), + ("version", models.CharField(max_length=255, null=True, blank=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), + ), + ( + "project", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ( + "tags", + taggit.managers.TaggableManager( + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + related_name="xforms", + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), + ), ], options={ - 'ordering': ('id_string',), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data')), + "ordering": ("id_string",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ), }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='xform', + name="xform", unique_together=set( - [('user', 'id_string', 'project'), - ('user', 'sms_id_string', 'project')]), + [("user", "id_string", "project"), ("user", "sms_id_string", "project")] + ), ), migrations.AlterUniqueTogether( - name='project', - unique_together=set([('name', 'organization')]), + name="project", + unique_together=set([("name", "organization")]), ), migrations.AddField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(to='logger.SurveyType', - on_delete=models.CASCADE), + model_name="instance", + name="survey_type", + field=models.ForeignKey(to="logger.SurveyType", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='tags', + model_name="instance", + name="tags", field=taggit.managers.TaggableManager( - to='taggit.Tag', through='taggit.TaggedItem', - help_text='A comma-separated list of tags.', - verbose_name='Tags'), + to="taggit.Tag", + through="taggit.TaggedItem", + help_text="A comma-separated list of tags.", + verbose_name="Tags", + ), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='user', + model_name="instance", + name="user", field=models.ForeignKey( - related_name='instances', to=settings.AUTH_USER_MODEL, + related_name="instances", + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, - null=True), + null=True, + ), preserve_default=True, ), migrations.AddField( - model_name='instance', - name='xform', + model_name="instance", + name="xform", field=models.ForeignKey( - related_name='instances', - to='logger.XForm', null=True, on_delete=models.CASCADE), + related_name="instances", + to="logger.XForm", + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), migrations.AddField( - model_name='dataview', - name='project', - field=models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE), + model_name="dataview", + name="project", + field=models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='dataview', - name='xform', - field=models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE), + model_name="dataview", + name="xform", + field=models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='instance', + model_name="attachment", + name="instance", field=models.ForeignKey( - related_name='attachments',to='logger.Instance', - on_delete=models.CASCADE), + related_name="attachments", + to="logger.Instance", + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py index aa0f1b0137..c29b565ef7 100644 --- a/onadata/apps/logger/migrations/0006_auto_20151106_0130.py +++ b/onadata/apps/logger/migrations/0006_auto_20151106_0130.py @@ -2,52 +2,67 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields import django.contrib.gis.db.models.fields class Migration(migrations.Migration): dependencies = [ - ('logger', '0005_auto_20151015_0758'), + ("logger", "0005_auto_20151015_0758"), ] operations = [ migrations.CreateModel( - name='OsmData', + name="OsmData", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('xml', models.TextField()), - ('osm_id', models.CharField(max_length=10)), - ('tags', jsonfield.fields.JSONField(default={})), - ('geom', - django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326)), - ('filename', models.CharField(max_length=255)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('instance', models.ForeignKey(related_name='osm_data', - to='logger.Instance', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("xml", models.TextField()), + ("osm_id", models.CharField(max_length=10)), + ("tags", models.JSONField(default={})), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326 + ), + ), + ("filename", models.CharField(max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "instance", + models.ForeignKey( + related_name="osm_data", + to="logger.Instance", + on_delete=models.CASCADE, + ), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('id_string',), 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form') - )}, + "ordering": ("id_string",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + }, ), ] diff --git a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py index bbb5f3bdfd..09c2af5b19 100644 --- a/onadata/apps/logger/migrations/0019_auto_20160307_0256.py +++ b/onadata/apps/logger/migrations/0019_auto_20160307_0256.py @@ -2,34 +2,36 @@ from __future__ import unicode_literals from django.db import models, migrations -import jsonfield.fields class Migration(migrations.Migration): dependencies = [ - ('logger', '0018_auto_20160301_0330'), + ("logger", "0018_auto_20160301_0330"), ] operations = [ migrations.AddField( - model_name='widget', - name='metadata', - field=jsonfield.fields.JSONField(default={}, blank=True), + model_name="widget", + name="metadata", + field=models.JSONField(default={}, blank=True), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(default='', max_length=249), + model_name="instance", + name="uuid", + field=models.CharField(default="", max_length=249), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', - to='logger.XForm', null=True, - on_delete=models.CASCADE), + model_name="instance", + name="xform", + field=models.ForeignKey( + related_name="instances", + to="logger.XForm", + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/main/migrations/0001_initial.py b/onadata/apps/main/migrations/0001_initial.py index b3faf5e95a..897498e945 100644 --- a/onadata/apps/main/migrations/0001_initial.py +++ b/onadata/apps/main/migrations/0001_initial.py @@ -3,7 +3,6 @@ from django.db import models, migrations import onadata.apps.main.models.meta_data -import jsonfield.fields import django.utils.timezone from django.conf import settings @@ -11,250 +10,401 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), - ('auth', '0001_initial'), + ("logger", "0001_initial"), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( - name='Audit', + name="Audit", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('json', jsonfield.fields.JSONField()), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("json", models.JSONField()), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='MetaData', + name="MetaData", fields=[ - ('id', models.AutoField( - verbose_name='ID', serialize=False, auto_created=True, - primary_key=True)), - ('data_type', models.CharField(max_length=255)), - ('data_value', models.CharField(max_length=255)), - ('data_file', - models.FileField( - null=True, - upload_to=onadata.apps.main.models.meta_data.upload_to, - blank=True)), - ('data_file_type', - models.CharField(max_length=255, null=True, blank=True)), - ('file_hash', - models.CharField(max_length=50, null=True, blank=True)), - ('xform', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("data_type", models.CharField(max_length=255)), + ("data_value", models.CharField(max_length=255)), + ( + "data_file", + models.FileField( + null=True, + upload_to=onadata.apps.main.models.meta_data.upload_to, + blank=True, + ), + ), + ( + "data_file_type", + models.CharField(max_length=255, null=True, blank=True), + ), + ("file_hash", models.CharField(max_length=50, null=True, blank=True)), + ( + "xform", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='TokenStorageModel', + name="TokenStorageModel", fields=[ - ('id', - models.ForeignKey( - related_name='google_id', primary_key=True, - serialize=False, to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE - )), - ('token', models.TextField()), + ( + "id", + models.ForeignKey( + related_name="google_id", + primary_key=True, + serialize=False, + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), + ("token", models.TextField()), ], - options={ - }, + options={}, bases=(models.Model,), ), migrations.CreateModel( - name='UserProfile', + name="UserProfile", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('name', models.CharField(max_length=255, blank=True)), - ('city', models.CharField(max_length=255, blank=True)), - ('country', models.CharField( - blank=True, max_length=2, - choices=[ - (b'AF', 'Afghanistan'), (b'AL', 'Albania'), - (b'DZ', 'Algeria'), (b'AS', 'American Samoa'), - (b'AD', 'Andorra'), (b'AO', 'Angola'), - (b'AI', 'Anguilla'), (b'AQ', 'Antarctica'), - (b'AG', 'Antigua and Barbuda'), (b'AR', 'Argentina'), - (b'AM', 'Armenia'), (b'AW', 'Aruba'), - (b'AU', 'Australia'), (b'AT', 'Austria'), - (b'AZ', 'Azerbaijan'), (b'BS', 'Bahamas'), - (b'BH', 'Bahrain'), (b'BD', 'Bangladesh'), - (b'BB', 'Barbados'), (b'BY', 'Belarus'), - (b'BE', 'Belgium'), (b'BZ', 'Belize'), - (b'BJ', 'Benin'), (b'BM', 'Bermuda'), - (b'BT', 'Bhutan'), (b'BO', 'Bolivia'), - (b'BQ', 'Bonaire, Sint Eustatius and Saba'), - (b'BA', 'Bosnia and Herzegovina'), (b'BW', 'Botswana'), - (b'BR', 'Brazil'), - (b'IO', 'British Indian Ocean Territory'), - (b'BN', 'Brunei Darussalam'), (b'BG', 'Bulgaria'), - (b'BF', 'Burkina Faso'), (b'BI', 'Burundi'), - (b'KH', 'Cambodia'), (b'CM', 'Cameroon'), - (b'CA', 'Canada'), (b'CV', 'Cape Verde'), - (b'KY', 'Cayman Islands'), - (b'CF', 'Central African Republic'), (b'TD', 'Chad'), - (b'CL', 'Chile'), (b'CN', 'China'), - (b'CX', 'Christmas Island'), - (b'CC', 'Cocos (Keeling) Islands'), - (b'CO', 'Colombia'), (b'KM', 'Comoros'), - (b'CG', 'Congo'), - (b'CD', 'Congo, The Democratic Republic of the'), - (b'CK', 'Cook Islands'), (b'CR', 'Costa Rica'), - (b'CI', 'Ivory Coast'), (b'HR', 'Croatia'), - (b'CU', 'Cuba'), (b'CW', 'Curacao'), (b'CY', 'Cyprus'), - (b'CZ', 'Czech Republic'), (b'DK', 'Denmark'), - (b'DJ', 'Djibouti'), (b'DM', 'Dominica'), - (b'DO', 'Dominican Republic'), (b'EC', 'Ecuador'), - (b'EG', 'Egypt'), (b'SV', 'El Salvador'), - (b'GQ', 'Equatorial Guinea'), (b'ER', 'Eritrea'), - (b'EE', 'Estonia'), (b'ET', 'Ethiopia'), - (b'FK', 'Falkland Islands (Malvinas)'), - (b'FO', 'Faroe Islands'), (b'FJ', 'Fiji'), - (b'FI', 'Finland'), (b'FR', 'France'), - (b'GF', 'French Guiana'), (b'PF', 'French Polynesia'), - (b'TF', 'French Southern Territories'), - (b'GA', 'Gabon'), (b'GM', 'Gambia'), - (b'GE', 'Georgia'), (b'DE', 'Germany'), - (b'GH', 'Ghana'), (b'GI', 'Gibraltar'), - (b'GR', 'Greece'), (b'GL', 'Greenland'), - (b'GD', 'Grenada'), (b'GP', 'Guadeloupe'), - (b'GU', 'Guam'), (b'GT', 'Guatemala'), - (b'GG', 'Guernsey'), (b'GN', 'Guinea'), - (b'GW', 'Guinea-Bissau'), (b'GY', 'Guyana'), - (b'HT', 'Haiti'), - (b'HM', 'Heard Island and McDonald Islands'), - (b'VA', 'Holy See (Vatican City State)'), - (b'HN', 'Honduras'), (b'HK', 'Hong Kong'), - (b'HU', 'Hungary'), (b'IS', 'Iceland'), - (b'IN', 'India'), (b'ID', 'Indonesia'), - (b'XZ', 'Installations in International Waters'), - (b'IR', 'Iran, Islamic Republic of'), (b'IQ', 'Iraq'), - (b'IE', 'Ireland'), (b'IM', 'Isle of Man'), - (b'IL', 'Israel'), (b'IT', 'Italy'), - (b'JM', 'Jamaica'), (b'JP', 'Japan'), - (b'JE', 'Jersey'), (b'JO', 'Jordan'), - (b'KZ', 'Kazakhstan'), (b'KE', 'Kenya'), - (b'KI', 'Kiribati'), - (b'KP', "Korea, Democratic People's Republic of"), - (b'KR', 'Korea, Republic of'), (b'XK', 'Kosovo'), - (b'KW', 'Kuwait'), (b'KG', 'Kyrgyzstan'), - (b'LA', "Lao People's Democratic Republic"), - (b'LV', 'Latvia'), (b'LB', 'Lebanon'), - (b'LS', 'Lesotho'), (b'LR', 'Liberia'), - (b'LY', 'Libyan Arab Jamahiriya'), - (b'LI', 'Liechtenstein'), (b'LT', 'Lithuania'), - (b'LU', 'Luxembourg'), (b'MO', 'Macao'), - (b'MK', 'Macedonia, The former Yugoslav Republic of'), - (b'MG', 'Madagascar'), (b'MW', 'Malawi'), - (b'MY', 'Malaysia'), (b'MV', 'Maldives'), - (b'ML', 'Mali'), (b'MT', 'Malta'), - (b'MH', 'Marshall Islands'), (b'MQ', 'Martinique'), - (b'MR', 'Mauritania'), (b'MU', 'Mauritius'), - (b'YT', 'Mayotte'), (b'MX', 'Mexico'), - (b'FM', 'Micronesia, Federated States of'), - (b'MD', 'Moldova, Republic of'), (b'MC', 'Monaco'), - (b'MN', 'Mongolia'), (b'ME', 'Montenegro'), - (b'MS', 'Montserrat'), (b'MA', 'Morocco'), - (b'MZ', 'Mozambique'), (b'MM', 'Myanmar'), - (b'NA', 'Namibia'), (b'NR', 'Nauru'), (b'NP', 'Nepal'), - (b'NL', 'Netherlands'), (b'NC', 'New Caledonia'), - (b'NZ', 'New Zealand'), (b'NI', 'Nicaragua'), - (b'NE', 'Niger'), (b'NG', 'Nigeria'), (b'NU', 'Niue'), - (b'NF', 'Norfolk Island'), - (b'MP', 'Northern Mariana Islands'), (b'NO', 'Norway'), - (b'OM', 'Oman'), (b'PK', 'Pakistan'), (b'PW', 'Palau'), - (b'PS', 'Palestinian Territory, Occupied'), - (b'PA', 'Panama'), (b'PG', 'Papua New Guinea'), - (b'PY', 'Paraguay'), (b'PE', 'Peru'), - (b'PH', 'Philippines'), (b'PN', 'Pitcairn'), - (b'PL', 'Poland'), (b'PT', 'Portugal'), - (b'PR', 'Puerto Rico'), (b'QA', 'Qatar'), - (b'RE', 'Reunion'), (b'RO', 'Romania'), - (b'RU', 'Russian Federation'), (b'RW', 'Rwanda'), - (b'SH', 'Saint Helena'), - (b'KN', 'Saint Kitts and Nevis'), - (b'LC', 'Saint Lucia'), - (b'PM', 'Saint Pierre and Miquelon'), - (b'VC', 'Saint Vincent and the Grenadines'), - (b'WS', 'Samoa'), (b'SM', 'San Marino'), - (b'ST', 'Sao Tome and Principe'), - (b'SA', 'Saudi Arabia'), (b'SN', 'Senegal'), - (b'RS', 'Serbia'), (b'SC', 'Seychelles'), - (b'SL', 'Sierra Leone'), (b'SG', 'Singapore'), - (b'SX', 'Sint Maarten (Dutch Part)'), - (b'SK', 'Slovakia'), (b'SI', 'Slovenia'), - (b'SB', 'Solomon Islands'), (b'SO', 'Somalia'), - (b'ZA', 'South Africa'), - (b'GS', - 'South Georgia and the South Sandwich Islands'), - (b'SS', 'South Sudan'), (b'ES', 'Spain'), - (b'LK', 'Sri Lanka'), (b'SD', 'Sudan'), - (b'SR', 'Suriname'), - (b'SJ', 'Svalbard and Jan Mayen'), - (b'SZ', 'Swaziland'), (b'SE', 'Sweden'), - (b'CH', 'Switzerland'), - (b'SY', 'Syrian Arab Republic'), - (b'TW', 'Taiwan, Province of China'), - (b'TJ', 'Tajikistan'), - (b'TZ', 'Tanzania, United Republic of'), - (b'TH', 'Thailand'), (b'TL', 'Timor-Leste'), - (b'TG', 'Togo'), (b'TK', 'Tokelau'), (b'TO', 'Tonga'), - (b'TT', 'Trinidad and Tobago'), (b'TN', 'Tunisia'), - (b'TR', 'Turkey'), (b'TM', 'Turkmenistan'), - (b'TC', 'Turks and Caicos Islands'), (b'TV', 'Tuvalu'), - (b'UG', 'Uganda'), (b'UA', 'Ukraine'), - (b'AE', 'United Arab Emirates'), - (b'GB', 'United Kingdom'), (b'US', 'United States'), - (b'UM', 'United States Minor Outlying Islands'), - (b'UY', 'Uruguay'), (b'UZ', 'Uzbekistan'), - (b'VU', 'Vanuatu'), (b'VE', 'Venezuela'), - (b'VN', 'Viet Nam'), - (b'VG', 'Virgin Islands, British'), - (b'VI', 'Virgin Islands, U.S.'), - (b'WF', 'Wallis and Futuna'), - (b'EH', 'Western Sahara'), (b'YE', 'Yemen'), - (b'ZM', 'Zambia'), (b'ZW', 'Zimbabwe'), - (b'ZZ', 'Unknown or unspecified country')])), - ('organization', models.CharField(max_length=255, blank=True)), - ('home_page', models.CharField(max_length=255, blank=True)), - ('twitter', models.CharField(max_length=255, blank=True)), - ('description', models.CharField(max_length=255, blank=True)), - ('require_auth', models.BooleanField( - default=False, - verbose_name='Require Phone Authentication')), - ('address', models.CharField(max_length=255, blank=True)), - ('phonenumber', models.CharField(max_length=30, blank=True)), - ('num_of_submissions', models.IntegerField(default=0)), - ('metadata', jsonfield.fields.JSONField(default={}, - blank=True)), - ('date_modified', models.DateTimeField( - default=django.utils.timezone.now, auto_now=True)), - ('created_by', models.ForeignKey( - blank=True, to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.SET_NULL)), - ('user', models.OneToOneField(related_name='profile', - to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("name", models.CharField(max_length=255, blank=True)), + ("city", models.CharField(max_length=255, blank=True)), + ( + "country", + models.CharField( + blank=True, + max_length=2, + choices=[ + (b"AF", "Afghanistan"), + (b"AL", "Albania"), + (b"DZ", "Algeria"), + (b"AS", "American Samoa"), + (b"AD", "Andorra"), + (b"AO", "Angola"), + (b"AI", "Anguilla"), + (b"AQ", "Antarctica"), + (b"AG", "Antigua and Barbuda"), + (b"AR", "Argentina"), + (b"AM", "Armenia"), + (b"AW", "Aruba"), + (b"AU", "Australia"), + (b"AT", "Austria"), + (b"AZ", "Azerbaijan"), + (b"BS", "Bahamas"), + (b"BH", "Bahrain"), + (b"BD", "Bangladesh"), + (b"BB", "Barbados"), + (b"BY", "Belarus"), + (b"BE", "Belgium"), + (b"BZ", "Belize"), + (b"BJ", "Benin"), + (b"BM", "Bermuda"), + (b"BT", "Bhutan"), + (b"BO", "Bolivia"), + (b"BQ", "Bonaire, Sint Eustatius and Saba"), + (b"BA", "Bosnia and Herzegovina"), + (b"BW", "Botswana"), + (b"BR", "Brazil"), + (b"IO", "British Indian Ocean Territory"), + (b"BN", "Brunei Darussalam"), + (b"BG", "Bulgaria"), + (b"BF", "Burkina Faso"), + (b"BI", "Burundi"), + (b"KH", "Cambodia"), + (b"CM", "Cameroon"), + (b"CA", "Canada"), + (b"CV", "Cape Verde"), + (b"KY", "Cayman Islands"), + (b"CF", "Central African Republic"), + (b"TD", "Chad"), + (b"CL", "Chile"), + (b"CN", "China"), + (b"CX", "Christmas Island"), + (b"CC", "Cocos (Keeling) Islands"), + (b"CO", "Colombia"), + (b"KM", "Comoros"), + (b"CG", "Congo"), + (b"CD", "Congo, The Democratic Republic of the"), + (b"CK", "Cook Islands"), + (b"CR", "Costa Rica"), + (b"CI", "Ivory Coast"), + (b"HR", "Croatia"), + (b"CU", "Cuba"), + (b"CW", "Curacao"), + (b"CY", "Cyprus"), + (b"CZ", "Czech Republic"), + (b"DK", "Denmark"), + (b"DJ", "Djibouti"), + (b"DM", "Dominica"), + (b"DO", "Dominican Republic"), + (b"EC", "Ecuador"), + (b"EG", "Egypt"), + (b"SV", "El Salvador"), + (b"GQ", "Equatorial Guinea"), + (b"ER", "Eritrea"), + (b"EE", "Estonia"), + (b"ET", "Ethiopia"), + (b"FK", "Falkland Islands (Malvinas)"), + (b"FO", "Faroe Islands"), + (b"FJ", "Fiji"), + (b"FI", "Finland"), + (b"FR", "France"), + (b"GF", "French Guiana"), + (b"PF", "French Polynesia"), + (b"TF", "French Southern Territories"), + (b"GA", "Gabon"), + (b"GM", "Gambia"), + (b"GE", "Georgia"), + (b"DE", "Germany"), + (b"GH", "Ghana"), + (b"GI", "Gibraltar"), + (b"GR", "Greece"), + (b"GL", "Greenland"), + (b"GD", "Grenada"), + (b"GP", "Guadeloupe"), + (b"GU", "Guam"), + (b"GT", "Guatemala"), + (b"GG", "Guernsey"), + (b"GN", "Guinea"), + (b"GW", "Guinea-Bissau"), + (b"GY", "Guyana"), + (b"HT", "Haiti"), + (b"HM", "Heard Island and McDonald Islands"), + (b"VA", "Holy See (Vatican City State)"), + (b"HN", "Honduras"), + (b"HK", "Hong Kong"), + (b"HU", "Hungary"), + (b"IS", "Iceland"), + (b"IN", "India"), + (b"ID", "Indonesia"), + (b"XZ", "Installations in International Waters"), + (b"IR", "Iran, Islamic Republic of"), + (b"IQ", "Iraq"), + (b"IE", "Ireland"), + (b"IM", "Isle of Man"), + (b"IL", "Israel"), + (b"IT", "Italy"), + (b"JM", "Jamaica"), + (b"JP", "Japan"), + (b"JE", "Jersey"), + (b"JO", "Jordan"), + (b"KZ", "Kazakhstan"), + (b"KE", "Kenya"), + (b"KI", "Kiribati"), + (b"KP", "Korea, Democratic People's Republic of"), + (b"KR", "Korea, Republic of"), + (b"XK", "Kosovo"), + (b"KW", "Kuwait"), + (b"KG", "Kyrgyzstan"), + (b"LA", "Lao People's Democratic Republic"), + (b"LV", "Latvia"), + (b"LB", "Lebanon"), + (b"LS", "Lesotho"), + (b"LR", "Liberia"), + (b"LY", "Libyan Arab Jamahiriya"), + (b"LI", "Liechtenstein"), + (b"LT", "Lithuania"), + (b"LU", "Luxembourg"), + (b"MO", "Macao"), + (b"MK", "Macedonia, The former Yugoslav Republic of"), + (b"MG", "Madagascar"), + (b"MW", "Malawi"), + (b"MY", "Malaysia"), + (b"MV", "Maldives"), + (b"ML", "Mali"), + (b"MT", "Malta"), + (b"MH", "Marshall Islands"), + (b"MQ", "Martinique"), + (b"MR", "Mauritania"), + (b"MU", "Mauritius"), + (b"YT", "Mayotte"), + (b"MX", "Mexico"), + (b"FM", "Micronesia, Federated States of"), + (b"MD", "Moldova, Republic of"), + (b"MC", "Monaco"), + (b"MN", "Mongolia"), + (b"ME", "Montenegro"), + (b"MS", "Montserrat"), + (b"MA", "Morocco"), + (b"MZ", "Mozambique"), + (b"MM", "Myanmar"), + (b"NA", "Namibia"), + (b"NR", "Nauru"), + (b"NP", "Nepal"), + (b"NL", "Netherlands"), + (b"NC", "New Caledonia"), + (b"NZ", "New Zealand"), + (b"NI", "Nicaragua"), + (b"NE", "Niger"), + (b"NG", "Nigeria"), + (b"NU", "Niue"), + (b"NF", "Norfolk Island"), + (b"MP", "Northern Mariana Islands"), + (b"NO", "Norway"), + (b"OM", "Oman"), + (b"PK", "Pakistan"), + (b"PW", "Palau"), + (b"PS", "Palestinian Territory, Occupied"), + (b"PA", "Panama"), + (b"PG", "Papua New Guinea"), + (b"PY", "Paraguay"), + (b"PE", "Peru"), + (b"PH", "Philippines"), + (b"PN", "Pitcairn"), + (b"PL", "Poland"), + (b"PT", "Portugal"), + (b"PR", "Puerto Rico"), + (b"QA", "Qatar"), + (b"RE", "Reunion"), + (b"RO", "Romania"), + (b"RU", "Russian Federation"), + (b"RW", "Rwanda"), + (b"SH", "Saint Helena"), + (b"KN", "Saint Kitts and Nevis"), + (b"LC", "Saint Lucia"), + (b"PM", "Saint Pierre and Miquelon"), + (b"VC", "Saint Vincent and the Grenadines"), + (b"WS", "Samoa"), + (b"SM", "San Marino"), + (b"ST", "Sao Tome and Principe"), + (b"SA", "Saudi Arabia"), + (b"SN", "Senegal"), + (b"RS", "Serbia"), + (b"SC", "Seychelles"), + (b"SL", "Sierra Leone"), + (b"SG", "Singapore"), + (b"SX", "Sint Maarten (Dutch Part)"), + (b"SK", "Slovakia"), + (b"SI", "Slovenia"), + (b"SB", "Solomon Islands"), + (b"SO", "Somalia"), + (b"ZA", "South Africa"), + (b"GS", "South Georgia and the South Sandwich Islands"), + (b"SS", "South Sudan"), + (b"ES", "Spain"), + (b"LK", "Sri Lanka"), + (b"SD", "Sudan"), + (b"SR", "Suriname"), + (b"SJ", "Svalbard and Jan Mayen"), + (b"SZ", "Swaziland"), + (b"SE", "Sweden"), + (b"CH", "Switzerland"), + (b"SY", "Syrian Arab Republic"), + (b"TW", "Taiwan, Province of China"), + (b"TJ", "Tajikistan"), + (b"TZ", "Tanzania, United Republic of"), + (b"TH", "Thailand"), + (b"TL", "Timor-Leste"), + (b"TG", "Togo"), + (b"TK", "Tokelau"), + (b"TO", "Tonga"), + (b"TT", "Trinidad and Tobago"), + (b"TN", "Tunisia"), + (b"TR", "Turkey"), + (b"TM", "Turkmenistan"), + (b"TC", "Turks and Caicos Islands"), + (b"TV", "Tuvalu"), + (b"UG", "Uganda"), + (b"UA", "Ukraine"), + (b"AE", "United Arab Emirates"), + (b"GB", "United Kingdom"), + (b"US", "United States"), + (b"UM", "United States Minor Outlying Islands"), + (b"UY", "Uruguay"), + (b"UZ", "Uzbekistan"), + (b"VU", "Vanuatu"), + (b"VE", "Venezuela"), + (b"VN", "Viet Nam"), + (b"VG", "Virgin Islands, British"), + (b"VI", "Virgin Islands, U.S."), + (b"WF", "Wallis and Futuna"), + (b"EH", "Western Sahara"), + (b"YE", "Yemen"), + (b"ZM", "Zambia"), + (b"ZW", "Zimbabwe"), + (b"ZZ", "Unknown or unspecified country"), + ], + ), + ), + ("organization", models.CharField(max_length=255, blank=True)), + ("home_page", models.CharField(max_length=255, blank=True)), + ("twitter", models.CharField(max_length=255, blank=True)), + ("description", models.CharField(max_length=255, blank=True)), + ( + "require_auth", + models.BooleanField( + default=False, verbose_name="Require Phone Authentication" + ), + ), + ("address", models.CharField(max_length=255, blank=True)), + ("phonenumber", models.CharField(max_length=30, blank=True)), + ("num_of_submissions", models.IntegerField(default=0)), + ("metadata", models.JSONField(default={}, blank=True)), + ( + "date_modified", + models.DateTimeField( + default=django.utils.timezone.now, auto_now=True + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.SET_NULL, + ), + ), + ( + "user", + models.OneToOneField( + related_name="profile", + to=settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + ), + ), ], options={ - 'permissions': ( - ('can_add_xform', - 'Can add/upload an xform to user profile'), - ('view_profile', 'Can view user profile')), + "permissions": ( + ("can_add_xform", "Can add/upload an xform to user profile"), + ("view_profile", "Can view user profile"), + ), }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='metadata', - unique_together=set([('xform', 'data_type', 'data_value')]), + name="metadata", + unique_together=set([("xform", "data_type", "data_value")]), ), ] - diff --git a/onadata/apps/viewer/migrations/0002_export_options.py b/onadata/apps/viewer/migrations/0002_export_options.py index 8da8deafe7..f7eb4f77d5 100644 --- a/onadata/apps/viewer/migrations/0002_export_options.py +++ b/onadata/apps/viewer/migrations/0002_export_options.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django.db import migrations -import jsonfield.fields +from django.db import models, migrations class Migration(migrations.Migration): dependencies = [ - ('viewer', '0001_initial'), + ("viewer", "0001_initial"), ] operations = [ migrations.AddField( - model_name='export', - name='options', - field=jsonfield.fields.JSONField(default={}), + model_name="export", + name="options", + field=models.JSONField(default={}), preserve_default=True, ), ] From f3bae6abc7b5746997839c7fede45851b891a9f3 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 26 Apr 2022 10:02:25 +0300 Subject: [PATCH 037/234] Fix regressions Signed-off-by: Kipchirchir Sigei --- .../libs/tests/utils/test_export_builder.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 5a750372d4..1f67fce10e 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -1617,10 +1617,10 @@ def test_to_xls_export_works(self): "children_cartoons", "children_cartoons_characters", ] - self.assertEqual(wb.sheet_names(), expected_sheet_names) + self.assertEqual(list(wb.get_sheet_names()), expected_sheet_names) # check header columns - main_sheet = wb.sheet_by_name("childrens_survey") + main_sheet = wb.get_sheet_by_name("childrens_survey") expected_column_headers = [ "name", "age", @@ -1644,10 +1644,10 @@ def test_to_xls_export_works(self): "_duration", "_submitted_by", ] - column_headers = main_sheet.row_values(0) - self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + column_headers = list(main_sheet.values)[0] + self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) - childrens_sheet = wb.sheet_by_name("children") + childrens_sheet = wb.get_sheet_by_name("children") expected_column_headers = [ "children/name", "children/age", @@ -1671,10 +1671,10 @@ def test_to_xls_export_works(self): "_duration", "_submitted_by", ] - column_headers = childrens_sheet.row_values(0) - self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + column_headers = list(childrens_sheet.values)[0] + self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) - cartoons_sheet = wb.sheet_by_name("children_cartoons") + cartoons_sheet = wb.get_sheet_by_name("children_cartoons") expected_column_headers = [ "children/cartoons/name", "children/cartoons/why", @@ -1690,10 +1690,10 @@ def test_to_xls_export_works(self): "_duration", "_submitted_by", ] - column_headers = cartoons_sheet.row_values(0) - self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + column_headers = list(cartoons_sheet.values)[0] + self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) - characters_sheet = wb.sheet_by_name("children_cartoons_characters") + characters_sheet = wb.get_sheet_by_name("children_cartoons_characters") expected_column_headers = [ "children/cartoons/characters/name", "children/cartoons/characters/good_or_evil", @@ -1709,8 +1709,8 @@ def test_to_xls_export_works(self): "_duration", "_submitted_by", ] - column_headers = characters_sheet.row_values(0) - self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + column_headers = list(characters_sheet.values)[0] + self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) xls_file.close() @@ -1726,7 +1726,7 @@ def test_to_xls_export_respects_custom_field_delimiter(self): wb = load_workbook(filename) # check header columns - main_sheet = wb.sheet_by_name("childrens_survey") + main_sheet = wb.get_sheet_by_name("childrens_survey") expected_column_headers = [ "name", "age", @@ -1750,8 +1750,8 @@ def test_to_xls_export_respects_custom_field_delimiter(self): "_duration", "_submitted_by", ] - column_headers = main_sheet.row_values(0) - self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) + column_headers = list(main_sheet.values)[0] + self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) xls_file.close() def test_get_valid_sheet_name_catches_duplicates(self): @@ -1800,7 +1800,7 @@ def test_to_xls_export_generates_valid_sheet_names(self): "childrens_survey_with_a_very_l2", "childrens_survey_with_a_very_l3", ] - self.assertEqual(wb.sheet_names(), expected_sheet_names) + self.assertEqual(list(wb.get_sheet_names()), expected_sheet_names) xls_file.close() def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): From 934c414e553fa1707cc3e0bc972826b153c370bf Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 26 Apr 2022 10:04:04 +0300 Subject: [PATCH 038/234] Update apps/api & apps/main modules Signed-off-by: Kipchirchir Sigei --- .../api/tests/viewsets/test_dataview_viewset.py | 3 +-- .../apps/api/tests/viewsets/test_xform_viewset.py | 4 ++-- onadata/apps/api/viewsets/dataview_viewset.py | 1 - onadata/apps/logger/models/data_view.py | 13 ++++++++++--- onadata/apps/logger/models/xform.py | 4 ++-- onadata/apps/main/models/audit.py | 2 +- onadata/apps/viewer/models/parsed_instance.py | 10 ++++++++-- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index 087e39d4f5..dd587ed5c9 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -496,12 +496,11 @@ def test_dataview_sql_injection(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) self.assertIn('detail', response.data) self.assertTrue(str(response.data.get('detail')) - .startswith("invalid input syntax for type integer")) + .startswith("invalid input syntax for integer")) def test_dataview_invalid_columns(self): data = { diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 37b8622934..ef921c6bf3 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -1974,8 +1974,8 @@ def test_form_clone_endpoint(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data["project"], - ["invalid literal for int() with base 10: 'abc123'"], + str(response.data["project"]), + '[ErrorDetail(string="Field \'id\' expected a number but got \'abc123\'.", code=\'invalid\')]', ) # pylint: disable=no-member diff --git a/onadata/apps/api/viewsets/dataview_viewset.py b/onadata/apps/api/viewsets/dataview_viewset.py index 9eb5aa426a..cf6a944d8f 100644 --- a/onadata/apps/api/viewsets/dataview_viewset.py +++ b/onadata/apps/api/viewsets/dataview_viewset.py @@ -86,7 +86,6 @@ def data(self, request, format="json", **kwargs): query = request.GET.get("query") export_type = self.kwargs.get("format", request.GET.get("format")) self.object = self.get_object() - if export_type is None or export_type in ["json", "debug"]: data = DataView.query_data( self.object, diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 3e79088249..a6a02d62e3 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -197,7 +197,7 @@ def has_instance(self, instance): cursor.execute(sql, [text(i) for i in params]) for row in cursor.fetchall(): - records = json.loads(row[0]) + records = row[0] return True if records else False @@ -266,6 +266,13 @@ def _get_where_clause( @classmethod def query_iterator(cls, sql, fields=None, params=[], count=False): + + def parse_json(data): + try: + return json.loads(data) + except ValueError: + return data + cursor = connection.cursor() sql_params = tuple(i if isinstance(i, tuple) else text(i) for i in params) @@ -284,14 +291,14 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): if fields is None: for row in cursor.fetchall(): - yield row[0] + yield parse_json(row[0]) else: if count: for row in cursor.fetchall(): yield dict(zip(fields, row)) else: for row in cursor.fetchall(): - yield dict(zip(fields, [json.loads(row[0]).get(f) for f in fields])) + yield dict(zip(fields, [parse_json(row[0]).get(f) for f in fields])) @classmethod def generate_query_string( diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index a47631e901..115d3a1b4e 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -19,7 +19,7 @@ from django.utils import timezone from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from six import iteritems, itervalues +from six import iteritems from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question @@ -394,7 +394,7 @@ def get_choice_label(self, field, choice_value, lang="English"): label = choice.label if isinstance(label, dict): - label = label.get(lang, itervalues(choice.label)[0]) + label = label.get(lang, list(choice.label.values())[0]) return label diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index ab0a7146c3..c6bf06e300 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -51,7 +51,7 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): if fields is None: for row in cursor.fetchall(): - yield json.loads(row[0]) + yield row[0] else: for row in cursor.fetchall(): yield dict(zip(fields, row)) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 6d6b70f2bc..6522527e5e 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -80,6 +80,13 @@ def _parse_sort_fields(fields): def _query_iterator(sql, fields=None, params=[], count=False): + + def parse_json(data): + try: + return json.loads(data) + except ValueError: + return data + if not sql: raise ValueError(_(u"Bad SQL: %s" % sql)) cursor = connection.cursor() @@ -92,10 +99,9 @@ def _query_iterator(sql, fields=None, params=[], count=False): fields = [u'count'] cursor.execute(sql, [text(i) for i in sql_params]) - if fields is None: for row in cursor.fetchall(): - yield json.loads(row[0]) + yield parse_json(row[0]) if row[0] else None else: for row in cursor.fetchall(): yield dict(zip(fields, row)) From db12a0f622ecf1a006c5762ecf76301fc0b34fc2 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 26 Apr 2022 15:55:32 +0300 Subject: [PATCH 039/234] Add tblib dependency to pickle tracebacks Signed-off-by: Kipchirchir Sigei --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47c823c506..f3b4b88aa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,14 +37,14 @@ jobs: - name: Setup Java uses: actions/setup-java@v2 with: - distribution: 'adopt' - java-version: '8' + distribution: "adopt" + java-version: "8" - name: Setup python uses: actions/setup-python@v2 with: python-version: 3.8 - architecture: 'x64' + architecture: "x64" - name: Get pip cache dir id: pip-cache @@ -71,6 +71,7 @@ jobs: pip install flake8 # Install jsonfield packages necessary for the Django <3 migrations pip install jsonfield + pip install tblib - name: Run tests run: | From dd6f8317236dde1d1b6a6fd44b3ddf27430a5459 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 26 Apr 2022 17:31:36 +0300 Subject: [PATCH 040/234] Json load cursor dicts Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/xform.py | 3 ++- onadata/apps/main/models/audit.py | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 115d3a1b4e..659091dc43 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -70,7 +70,8 @@ title_pattern = re.compile(r"(.*?)") -def cmp(x, y): (x > y) - (x < y) +def cmp(x, y): + return (x > y) - (x < y) def question_types_to_exclude(_type): diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index c6bf06e300..445d7054a9 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -32,6 +32,14 @@ def save(self): @classmethod def query_iterator(cls, sql, fields=None, params=[], count=False): + # cursor seems to stringify dicts + # added workaround to parse stringified dicts to json + def parse_json(data): + try: + return json.loads(data) + except ValueError: + return data + cursor = connection.cursor() sql_params = fields + params if fields is not None else params @@ -51,7 +59,7 @@ def query_iterator(cls, sql, fields=None, params=[], count=False): if fields is None: for row in cursor.fetchall(): - yield row[0] + yield parse_json(row[0]) else: for row in cursor.fetchall(): yield dict(zip(fields, row)) From 18cc28c9c5291e0ca6e8c7784fe64fa70d3a9067 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Tue, 26 Apr 2022 18:18:50 +0300 Subject: [PATCH 041/234] Handle non existent exports Signed-off-by: Kipchirchir Sigei --- onadata/libs/utils/export_tools.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index dc12d5ddee..37b7e405d3 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -495,7 +495,14 @@ def get_or_create_export_object(export_id, options, xform, export_type): :return: A new or found export object """ if export_id and Export.objects.filter(pk=export_id).exists(): - export = Export.objects.get(id=export_id) + try: + export = Export.objects.get(id=export_id) + except Export.DoesNotExist: + with use_master: + try: + return Export.objects.get(pk=export_id) + except Export.DoesNotExist: + pass else: export_options = get_export_options(options) export = Export.objects.create( From a8eaf91eb30be7410c06e5bacb7ac53533416d3a Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 27 Apr 2022 13:24:08 +0300 Subject: [PATCH 042/234] use setup-python@v3 on CI Signed-off-by: Kipchirchir Sigei --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3b4b88aa7..9949f8d88c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,9 @@ jobs: java-version: "8" - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: "3.x" architecture: "x64" - name: Get pip cache dir From 9e73e2a216a1388d31f0ada1a6bbe8578ef949c8 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 27 Apr 2022 13:57:37 +0300 Subject: [PATCH 043/234] update CI python version to 3.9 Signed-off-by: Kipchirchir Sigei --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9949f8d88c..69fc6a8231 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Setup python uses: actions/setup-python@v3 with: - python-version: "3.x" + python-version: 3.9 architecture: "x64" - name: Get pip cache dir From c3d754bac2dc29286572bb5ba9d010096f82069c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Apr 2022 06:01:46 +0300 Subject: [PATCH 044/234] jsonfield in CI not required --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69fc6a8231..fcc2e60ecc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,8 +69,6 @@ jobs: pip install -U pip pip install -r requirements/base.pip pip install flake8 - # Install jsonfield packages necessary for the Django <3 migrations - pip install jsonfield pip install tblib - name: Run tests From c8528768d18edb57e2a224a16ef578ec61aa868d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Apr 2022 14:12:09 +0300 Subject: [PATCH 045/234] Skip savReaderWriter tests --- onadata/libs/tests/utils/test_export_tools.py | 498 +++++++++--------- 1 file changed, 253 insertions(+), 245 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_tools.py b/onadata/libs/tests/utils/test_export_tools.py index 3230c2274d..81bc42ca6d 100644 --- a/onadata/libs/tests/utils/test_export_tools.py +++ b/onadata/libs/tests/utils/test_export_tools.py @@ -8,6 +8,7 @@ import zipfile from builtins import open from datetime import date, datetime, timedelta +from unittest import skipIf from django.conf import settings from django.contrib.sites.models import Site @@ -18,7 +19,11 @@ from pyxform.builder import create_survey_from_xls from rest_framework import exceptions from rest_framework.authtoken.models import Token -from savReaderWriter import SavWriter + +try: + from savReaderWriter import SavWriter +except ImportError: + SavWriter = None from onadata.apps.api import tests as api_tests from onadata.apps.logger.models import Attachment, Instance, XForm @@ -26,86 +31,87 @@ from onadata.apps.viewer.models.export import Export from onadata.apps.viewer.models.parsed_instance import query_data from onadata.apps.api.viewsets.data_viewset import DataViewSet -from onadata.libs.serializers.merged_xform_serializer import \ - MergedXFormSerializer -from onadata.libs.utils.export_builder import (encode_if_str, - get_value_or_attachment_uri) +from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer +from onadata.libs.utils.export_builder import encode_if_str, get_value_or_attachment_uri from onadata.libs.utils.export_tools import ( - ExportBuilder, check_pending_export, generate_attachments_zip_export, - generate_export, generate_kml_export, generate_osm_export, - get_repeat_index_tags, kml_export_data, parse_request_export_options, - should_create_new_export, str_to_bool) + ExportBuilder, + check_pending_export, + generate_attachments_zip_export, + generate_export, + generate_kml_export, + generate_osm_export, + get_repeat_index_tags, + kml_export_data, + parse_request_export_options, + should_create_new_export, + str_to_bool, +) def _logger_fixture_path(*args): - return os.path.join(settings.PROJECT_ROOT, 'libs', 'tests', 'fixtures', - *args) + return os.path.join(settings.PROJECT_ROOT, "libs", "tests", "fixtures", *args) class TestExportTools(TestBase): """ Test export_tools functions. """ + def _create_old_export(self, xform, export_type, options): Export(xform=xform, export_type=export_type, options=options).save() - self.export = Export.objects.filter( - xform=xform, export_type=export_type) + self.export = Export.objects.filter(xform=xform, export_type=export_type) def test_encode_if_str(self): row = {"date": date(1899, 9, 9)} date_str = encode_if_str(row, "date", True) - self.assertEqual(date_str, '1899-09-09') + self.assertEqual(date_str, "1899-09-09") row = {"date": date(2001, 9, 9)} date_str = encode_if_str(row, "date", True) - self.assertEqual(date_str, '2001-09-09') + self.assertEqual(date_str, "2001-09-09") row = {"datetime": datetime(1899, 9, 9)} date_str = encode_if_str(row, "datetime", True) - self.assertEqual(date_str, '1899-09-09T00:00:00') + self.assertEqual(date_str, "1899-09-09T00:00:00") row = {"datetime": datetime(2001, 9, 9)} date_str = encode_if_str(row, "datetime", True) - self.assertEqual(date_str, '2001-09-09T00:00:00') + self.assertEqual(date_str, "2001-09-09T00:00:00") row = {"integer_value": 1} - integer_value = encode_if_str( - row, "integer_value", sav_writer=True) - self.assertEqual(integer_value, '1') + integer_value = encode_if_str(row, "integer_value", sav_writer=True) + self.assertEqual(integer_value, "1") - integer_value = encode_if_str( - row, "integer_value") + integer_value = encode_if_str(row, "integer_value") self.assertEqual(integer_value, 1) def test_generate_osm_export(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') - combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") + combined_osm_path = os.path.join(osm_fixtures_dir, "combined.osm") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = {"extension": Attachment.OSM} - export = generate_osm_export(Attachment.OSM, self.user.username, - self.xform.id_string, None, options) + export = generate_osm_export( + Attachment.OSM, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) - with open(combined_osm_path, encoding='utf-8') as f: + with open(combined_osm_path, encoding="utf-8") as f: osm = f.read() with default_storage.open(export.filepath) as f2: - content = f2.read().decode('utf-8') + content = f2.read().decode("utf-8") self.assertMultiLineEqual(content.strip(), osm.strip()) # delete submission and check that content is no longer in export @@ -113,37 +119,35 @@ def test_generate_osm_export(self): submission.deleted_at = timezone.now() submission.save() - export = generate_osm_export(Attachment.OSM, self.user.username, - self.xform.id_string, None, options) + export = generate_osm_export( + Attachment.OSM, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) with default_storage.open(export.filepath) as f2: content = f2.read() - self.assertEqual(content, b'') + self.assertEqual(content, b"") def test_generate_attachments_zip_export(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = {"extension": Export.ZIP_EXPORT} export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) @@ -153,8 +157,7 @@ def test_generate_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # deleted submission @@ -163,8 +166,8 @@ def test_generate_attachments_zip_export(self): submission.save() export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") @@ -172,8 +175,7 @@ def test_generate_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertFalse( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertFalse(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) def test_should_create_new_export(self): @@ -184,7 +186,8 @@ def test_should_create_new_export(self): self._publish_transportation_form_and_submit_instance() will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) @@ -192,7 +195,8 @@ def test_should_create_new_export(self): # deleted self.xform.instances.first().delete() will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) def test_should_create_export_when_submission_deleted(self): @@ -205,44 +209,45 @@ def test_should_create_export_when_submission_deleted(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } - submission_count = self.xform.instances.filter( - deleted_at__isnull=True).count() + submission_count = self.xform.instances.filter(deleted_at__isnull=True).count() self._create_old_export(self.xform, export_type, options) # Delete submission instance = self.xform.instances.first() instance.set_deleted(datetime.now(), self.user) self.assertEqual( - submission_count - 1, self.xform.instances.filter( - deleted_at__isnull=True).count()) + submission_count - 1, + self.xform.instances.filter(deleted_at__isnull=True).count(), + ) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) self._create_old_export(self.xform, export_type, options) # Deleting submission via the API still triggers a new export # when requested - instance_id = self.xform.instances.filter( - deleted_at__isnull=True).first().id - view = DataViewSet.as_view( - {'delete': 'destroy'} - ) + instance_id = self.xform.instances.filter(deleted_at__isnull=True).first().id + view = DataViewSet.as_view({"delete": "destroy"}) token = Token.objects.get(user=self.user) - data = {'instance_ids': [instance_id]} + data = {"instance_ids": [instance_id]} request = self.factory.delete( - '/', data=data, HTTP_AUTHORIZATION=f'Token {token}') + "/", data=data, HTTP_AUTHORIZATION=f"Token {token}" + ) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) self.assertEqual( - submission_count - 2, self.xform.instances.filter( - deleted_at__isnull=True).count()) + submission_count - 2, + self.xform.instances.filter(deleted_at__isnull=True).count(), + ) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) @@ -252,12 +257,13 @@ def test_should_not_create_new_export_when_old_exists(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._create_old_export(self.xform, export_type, options) will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertFalse(will_create_new_export) @@ -266,164 +272,174 @@ def test_should_create_new_export_when_filter_defined(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) # Call should_create_new_export with updated options - options['remove_group_name'] = True + options["remove_group_name"] = True will_create_new_export = should_create_new_export( - self.xform, export_type, options) + self.xform, export_type, options + ) self.assertTrue(will_create_new_export) def test_get_value_or_attachment_uri(self): path = os.path.join( - os.path.dirname(__file__), 'fixtures', - 'photo_type_in_repeat_group.xlsx') + os.path.dirname(__file__), "fixtures", "photo_type_in_repeat_group.xlsx" + ) self._publish_xls_file_and_set_xform(path) - filename = u'bob/attachments/123.jpg' - download_url = u'/api/v1/files/1?filename=%s' % filename + filename = "bob/attachments/123.jpg" + download_url = "/api/v1/files/1?filename=%s" % filename # used a smaller version of row because we only using _attachmets key row = { - u'_attachments': [{ - u'mimetype': u'image/jpeg', - u'medium_download_url': u'%s&suffix=medium' % download_url, - u'download_url': download_url, - u'filename': filename, - u'name': '123.jpg', - u'instance': 1, - u'small_download_url': u'%s&suffix=small' % download_url, - u'id': 1, - u'xform': 1 - }] + "_attachments": [ + { + "mimetype": "image/jpeg", + "medium_download_url": "%s&suffix=medium" % download_url, + "download_url": download_url, + "filename": filename, + "name": "123.jpg", + "instance": 1, + "small_download_url": "%s&suffix=small" % download_url, + "id": 1, + "xform": 1, + } + ] } # yapf: disable # when include_images is True, you get the attachment url - media_xpaths = ['photo'] + media_xpaths = ["photo"] attachment_list = None - key = 'photo' - value = u'123.jpg' - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + key = "photo" + value = "123.jpg" + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) current_site = Site.objects.get_current() - url = 'http://%s%s' % (current_site.domain, download_url) + url = "http://%s%s" % (current_site.domain, download_url) self.assertEqual(url, val_or_url) # when include_images is False, you get the value media_xpaths = [] - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) self.assertEqual(value, val_or_url) # test that when row is an empty dict, the function still returns a # value - row.pop('_attachments', None) + row.pop("_attachments", None) self.assertEqual(row, {}) - media_xpaths = ['photo'] - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + media_xpaths = ["photo"] + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) self.assertEqual(value, val_or_url) def test_get_attachment_uri_for_filename_with_space(self): path = os.path.join( - os.path.dirname(__file__), 'fixtures', - 'photo_type_in_repeat_group.xlsx') + os.path.dirname(__file__), "fixtures", "photo_type_in_repeat_group.xlsx" + ) self._publish_xls_file_and_set_xform(path) - filename = u'bob/attachments/1_2_3.jpg' - download_url = u'/api/v1/files/1?filename=%s' % filename + filename = "bob/attachments/1_2_3.jpg" + download_url = "/api/v1/files/1?filename=%s" % filename # used a smaller version of row because we only using _attachmets key row = { - u'_attachments': [{ - u'mimetype': u'image/jpeg', - u'medium_download_url': u'%s&suffix=medium' % download_url, - u'download_url': download_url, - u'filename': filename, - u'name': '1 2 3.jpg', - u'instance': 1, - u'small_download_url': u'%s&suffix=small' % download_url, - u'id': 1, - u'xform': 1 - }] + "_attachments": [ + { + "mimetype": "image/jpeg", + "medium_download_url": "%s&suffix=medium" % download_url, + "download_url": download_url, + "filename": filename, + "name": "1 2 3.jpg", + "instance": 1, + "small_download_url": "%s&suffix=small" % download_url, + "id": 1, + "xform": 1, + } + ] } # yapf: disable # when include_images is True, you get the attachment url - media_xpaths = ['photo'] + media_xpaths = ["photo"] attachment_list = None - key = 'photo' - value = u'1 2 3.jpg' - val_or_url = get_value_or_attachment_uri(key, value, row, self.xform, - media_xpaths, attachment_list) + key = "photo" + value = "1 2 3.jpg" + val_or_url = get_value_or_attachment_uri( + key, value, row, self.xform, media_xpaths, attachment_list + ) self.assertTrue(val_or_url) current_site = Site.objects.get_current() - url = 'http://%s%s' % (current_site.domain, download_url) + url = "http://%s%s" % (current_site.domain, download_url) self.assertEqual(url, val_or_url) def test_parse_request_export_options(self): request = self.factory.get( - '/export_async', + "/export_async", data={ "binary_select_multiples": "true", "do_not_split_select_multiples": "false", "remove_group_name": "false", "include_labels": "false", "include_labels_only": "false", - "include_images": "false" - }) + "include_images": "false", + }, + ) options = parse_request_export_options(request.GET) - self.assertEqual(options['split_select_multiples'], True) - self.assertEqual(options['binary_select_multiples'], True) - self.assertEqual(options['include_labels'], False) - self.assertEqual(options['include_labels_only'], False) - self.assertEqual(options['remove_group_name'], False) - self.assertEqual(options['include_images'], False) + self.assertEqual(options["split_select_multiples"], True) + self.assertEqual(options["binary_select_multiples"], True) + self.assertEqual(options["include_labels"], False) + self.assertEqual(options["include_labels_only"], False) + self.assertEqual(options["remove_group_name"], False) + self.assertEqual(options["include_images"], False) request = self.factory.get( - '/export_async', + "/export_async", data={ "do_not_split_select_multiples": "true", "remove_group_name": "true", "include_labels": "true", "include_labels_only": "true", - "include_images": "true" - }) + "include_images": "true", + }, + ) options = parse_request_export_options(request.GET) - self.assertEqual(options['split_select_multiples'], False) - self.assertEqual(options['include_labels'], True) - self.assertEqual(options['include_labels_only'], True) - self.assertEqual(options['remove_group_name'], True) - self.assertEqual(options['include_images'], True) + self.assertEqual(options["split_select_multiples"], False) + self.assertEqual(options["include_labels"], True) + self.assertEqual(options["include_labels_only"], True) + self.assertEqual(options["remove_group_name"], True) + self.assertEqual(options["include_images"], True) def test_export_not_found(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) - export = Export( - xform=self.xform, export_type=export_type, options=options) + export = Export(xform=self.xform, export_type=export_type, options=options) export.save() export_id = export.pk @@ -449,47 +465,45 @@ def test_kml_export_data(self): | | fruits | orange | Orange | | | fruits | mango | Mango | """ - xform1 = self._publish_markdown(kml_md, self.user, id_string='a') - xform2 = self._publish_markdown(kml_md, self.user, id_string='b') + xform1 = self._publish_markdown(kml_md, self.user, id_string="a") + xform2 = self._publish_markdown(kml_md, self.user, id_string="b") xml = '-1.28 36.83orange' Instance(xform=xform1, xml=xml).save() xml = '32.85 13.04mango' Instance(xform=xform2, xml=xml).save() data = { - 'xforms': [ + "xforms": [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], - 'name': 'Merged Dataset', - 'project': - "http://testserver/api/v1/projects/%s" % xform1.project.pk, + "name": "Merged Dataset", + "project": "http://testserver/api/v1/projects/%s" % xform1.project.pk, } # yapf: disable - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user - serializer = MergedXFormSerializer( - data=data, context={'request': request}) + serializer = MergedXFormSerializer(data=data, context={"request": request}) self.assertTrue(serializer.is_valid()) serializer.save() - xform = XForm.objects.filter( - pk__gt=xform2.pk, is_merged_dataset=True).first() - expected_data = [{ - 'name': u'a', - 'image_urls': [], - 'lat': -1.28, - 'table': u'
    GPS-1.28 36.83
    Fruitorange
    ', # noqa pylint: disable=C0301 - 'lng': 36.83, - 'id': xform1.instances.all().first().pk - }, { - 'name': u'b', - 'image_urls': [], - 'lat': 32.85, - 'table': - u'
    GPS32.85 13.04
    Fruitmango
    ', # noqa pylint: disable=C0301 - 'lng': 13.04, - 'id': xform2.instances.all().first().pk - }] # yapf: disable - self.assertEqual( - kml_export_data(xform.id_string, xform.user), expected_data) + xform = XForm.objects.filter(pk__gt=xform2.pk, is_merged_dataset=True).first() + expected_data = [ + { + "name": "a", + "image_urls": [], + "lat": -1.28, + "table": '
    GPS-1.28 36.83
    Fruitorange
    ', # noqa pylint: disable=C0301 + "lng": 36.83, + "id": xform1.instances.all().first().pk, + }, + { + "name": "b", + "image_urls": [], + "lat": 32.85, + "table": '
    GPS32.85 13.04
    Fruitmango
    ', # noqa pylint: disable=C0301 + "lng": 13.04, + "id": xform2.instances.all().first().pk, + }, + ] # yapf: disable + self.assertEqual(kml_export_data(xform.id_string, xform.user), expected_data) def test_kml_exports(self): """ @@ -500,15 +514,14 @@ def test_kml_exports(self): "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True, - "extension": 'kml' + "extension": "kml", } self._publish_transportation_form_and_submit_instance() username = self.xform.user.username id_string = self.xform.id_string - export = generate_kml_export( - export_type, username, id_string, options=options) + export = generate_kml_export(export_type, username, id_string, options=options) self.assertIsNotNone(export) self.assertTrue(export.is_successful) @@ -517,31 +530,28 @@ def test_kml_exports(self): export.delete() export = generate_kml_export( - export_type, - username, - id_string, - export_id=export_id, - options=options) + export_type, username, id_string, export_id=export_id, options=options + ) self.assertIsNotNone(export) self.assertTrue(export.is_successful) def test_str_to_bool(self): self.assertTrue(str_to_bool(True)) - self.assertTrue(str_to_bool('True')) - self.assertTrue(str_to_bool('TRUE')) - self.assertTrue(str_to_bool('true')) - self.assertTrue(str_to_bool('t')) - self.assertTrue(str_to_bool('1')) + self.assertTrue(str_to_bool("True")) + self.assertTrue(str_to_bool("TRUE")) + self.assertTrue(str_to_bool("true")) + self.assertTrue(str_to_bool("t")) + self.assertTrue(str_to_bool("1")) self.assertTrue(str_to_bool(1)) self.assertFalse(str_to_bool(False)) - self.assertFalse(str_to_bool('False')) - self.assertFalse(str_to_bool('F')) - self.assertFalse(str_to_bool('random')) + self.assertFalse(str_to_bool("False")) + self.assertFalse(str_to_bool("F")) + self.assertFalse(str_to_bool("random")) self.assertFalse(str_to_bool(234)) self.assertFalse(str_to_bool(0)) - self.assertFalse(str_to_bool('0')) + self.assertFalse(str_to_bool("0")) def test_get_sav_value_labels(self): md = """ @@ -560,7 +570,7 @@ def test_get_sav_value_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_sav_choice_list_with_missing_values(self): @@ -580,7 +590,7 @@ def test_sav_choice_list_with_missing_values(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': ''}} + expected_data = {"fruit": {"orange": "Orange", "mango": ""}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_get_sav_value_labels_multi_language(self): @@ -600,11 +610,11 @@ def test_get_sav_value_labels_multi_language(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) - export_builder.dd._default_language = 'Swahili' - expected_data = {'fruit': {'orange': 'Chungwa', 'mango': 'Maembe'}} + export_builder.dd._default_language = "Swahili" + expected_data = {"fruit": {"orange": "Chungwa", "mango": "Maembe"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) def test_get_sav_value_labels_for_choice_filter(self): @@ -624,12 +634,14 @@ def test_get_sav_value_labels_for_choice_filter(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - expected_data = {'fruit': {'orange': 'Orange', 'mango': 'Mango'}} + expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) + @skipIf(SavWriter is None, "savReaderWriter not supported now.") def test_sav_duplicate_columns(self): - more_than_64_char = "akjasdlsakjdkjsadlsakjgdlsagdgdgdsajdgkjdsdgsj" \ - "adsasdasgdsahdsahdsadgsdf" + more_than_64_char = ( + "akjasdlsakjdkjsadlsakjgdlsagdgdgdsajdgkjdsdgsj" "adsasdasgdsahdsahdsadgsdf" + ) md = """ | survey | | | type | name | label | choice_filter | @@ -651,8 +663,9 @@ def test_sav_duplicate_columns(self): | | fts | orange | Orange | 1 | | | fts | mango | Mango | 1 | """ - md = md.format(more_than_64_char, more_than_64_char, more_than_64_char, - more_than_64_char) + md = md.format( + more_than_64_char, more_than_64_char, more_than_64_char, more_than_64_char + ) survey = self.md_to_pyxform_survey(md) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True @@ -661,14 +674,14 @@ def test_sav_duplicate_columns(self): export_builder.set_survey(survey) for sec in export_builder.sections: - sav_options = export_builder._get_sav_options(sec['elements']) + sav_options = export_builder._get_sav_options(sec["elements"]) sav_file = NamedTemporaryFile(suffix=".sav") # No exception is raised SavWriter(sav_file.name, **sav_options) + @skipIf(SavWriter is None, "savReaderWriter not supported now.") def test_sav_special_char_columns(self): - survey = create_survey_from_xls( - _logger_fixture_path('grains/grains.xlsx')) + survey = create_survey_from_xls(_logger_fixture_path("grains/grains.xlsx")) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) @@ -676,7 +689,7 @@ def test_sav_special_char_columns(self): export_builder.set_survey(survey) for sec in export_builder.sections: - sav_options = export_builder._get_sav_options(sec['elements']) + sav_options = export_builder._get_sav_options(sec["elements"]) sav_file = NamedTemporaryFile(suffix=".sav") # No exception is raised SavWriter(sav_file.name, **sav_options) @@ -690,7 +703,8 @@ def test_retrieving_pending_export(self): xform=self.xform, export_type=Export.CSV_EXPORT, options={}, - task_id="abcsde") + task_id="abcsde", + ) export.save() @@ -715,43 +729,40 @@ def test_get_repeat_index_tags(self): """ self.assertIsNone(get_repeat_index_tags(None)) - self.assertEqual(get_repeat_index_tags('.'), ('.', '.')) - self.assertEqual(get_repeat_index_tags('{,}'), ('{', '}')) + self.assertEqual(get_repeat_index_tags("."), (".", ".")) + self.assertEqual(get_repeat_index_tags("{,}"), ("{", "}")) with self.assertRaises(exceptions.ParseError): - get_repeat_index_tags('p') + get_repeat_index_tags("p") def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] osm_fixtures_dir = os.path.realpath( - os.path.join( - os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) for filename in filenames - ] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + os.path.join(os.path.dirname(api_tests.__file__), "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_file_and_set_xform(xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + count = Attachment.objects.filter(extension="osm").count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) options = { "extension": Export.ZIP_EXPORT, - "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'} + "query": '{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}', + } filter_query = options.get("query") - instance_ids = query_data( - self.xform, fields='["_id"]', query=filter_query) + instance_ids = query_data(self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export.is_successful) @@ -761,22 +772,20 @@ def test_generate_filtered_attachments_zip_export(self): zip_file.close() filtered_attachments = Attachment.objects.filter( - instance__xform_id=self.xform.pk).filter( - instance_id__in=[i_id['_id'] for i_id in instance_ids]) + instance__xform_id=self.xform.pk + ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) - self.assertNotEqual( - Attachment.objects.count(), filtered_attachments.count()) + self.assertNotEqual(Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query - options.pop('query') + options.pop("query") export1 = generate_attachments_zip_export( - Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, - options) + Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options + ) self.assertTrue(export1.is_successful) @@ -786,6 +795,5 @@ def test_generate_filtered_attachments_zip_export(self): zip_file.close() for a in Attachment.objects.all(): - self.assertTrue( - os.path.exists(os.path.join(temp_dir, a.media_file.name))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) From a8df483fabbf1f2bf6fbaf4b342de860249595f0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Apr 2022 16:42:09 +0300 Subject: [PATCH 046/234] Handle "Attribute etag_hash defined outside __init__" --- onadata/apps/api/viewsets/data_viewset.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 47b205a123..f6dcba0e54 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -272,7 +272,7 @@ def labels(self, request, *args, **kwargs): if request.method == "GET": http_status = status.HTTP_200_OK - self.etag_data = data + setattr(self, "etag_data", data) return Response(data, status=http_status) @@ -308,7 +308,7 @@ def enketo(self, request, *args, **kwargs): else: raise PermissionDenied(_("You do not have edit permissions.")) - self.etag_data = data + setattr(self, "etag_data", data) return Response(data=data) @@ -631,7 +631,9 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): # the configured SUBMISSION_RETRIEVAL_THRESHOLD setting if enable_etag: if isinstance(self.object_list, QuerySet): - self.etag_hash = get_etag_hash_from_query(self.object_list) + setattr( + self, "etag_hash", (get_etag_hash_from_query(self.object_list)) + ) else: sql, params, records = get_sql_with_params( xform, @@ -641,7 +643,11 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): limit=limit, fields=fields, ) - self.etag_hash = get_etag_hash_from_query(records, sql, params) + setattr( + self, + "etag_hash", + (get_etag_hash_from_query(records, sql, params)), + ) except ValueError as e: raise ParseError(text(e)) except DataError as e: From 95876359b7669276cd17b686ba621f95e5bd1304 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Apr 2022 19:05:21 +0300 Subject: [PATCH 047/234] Cleanup onadata/settings module --- onadata/settings/common.py | 25 +++---- onadata/settings/debug_toolbar_settings.py | 28 ++++--- onadata/settings/default_settings.py | 24 +++--- onadata/settings/docker.py | 87 +++++++++++----------- onadata/settings/drone_test.py | 49 ++++++------ onadata/settings/github_actions_test.py | 76 +++++++++---------- onadata/settings/production_example.py | 69 +++++++++-------- onadata/settings/staging_example.py | 55 +++++++------- onadata/settings/travis_test.py | 82 ++++++++++---------- 9 files changed, 257 insertions(+), 238 deletions(-) diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 665dc8ea83..97f7764c68 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -1,4 +1,7 @@ # vim: set fileencoding=utf-8 +""" +Base Django settings module. +""" # this system uses structured settings as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # @@ -12,9 +15,8 @@ import logging import os import socket -import subprocess # noqa, used by included files import sys -from imp import reload +from importlib import reload from celery.signals import after_setup_logger from django.core.exceptions import SuspiciousOperation @@ -153,7 +155,7 @@ }, ] -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" MIDDLEWARE = ( "onadata.libs.profiling.sql.SqlTimingMiddleware", @@ -423,6 +425,9 @@ def skip_suspicious_operations(record): def configure_logging(logger, **kwargs): + """ + Add AdminEmailHandler to the logger + """ admin_email_handler = AdminEmailHandler() admin_email_handler.setLevel(logging.ERROR) logger.addHandler(admin_email_handler) @@ -540,17 +545,14 @@ def configure_logging(logger, **kwargs): EXPORT_WITH_IMAGE_DEFAULT = True try: - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: RESERVED_USERNAMES = [line.rstrip() for line in f] except EnvironmentError: RESERVED_USERNAMES = [] STATIC_DOC = "/static/docs/index.html" -try: - HOSTNAME = socket.gethostname() -except Exception: - HOSTNAME = "localhost" +HOSTNAME = socket.gethostname() CACHE_MIXIN_SECONDS = 60 @@ -561,13 +563,6 @@ def configure_logging(logger, **kwargs): DEFAULT_CELERY_INTERVAL_MAX = 0.5 DEFAULT_CELERY_INTERVAL_STEP = 0.5 -# legacy setting for old sites who still use a local_settings.py file and have -# not updated to presets/ -try: - from local_settings import * # noqa -except ImportError: - pass - # email verification ENABLE_EMAIL_VERIFICATION = False VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" diff --git a/onadata/settings/debug_toolbar_settings.py b/onadata/settings/debug_toolbar_settings.py index 986ee81421..60c71476e3 100644 --- a/onadata/settings/debug_toolbar_settings.py +++ b/onadata/settings/debug_toolbar_settings.py @@ -1,33 +1,37 @@ +""" +Django debug toolbar example settings module. +""" # this system uses structured settings.py as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # -# this third-level staging file overrides some definitions in staging.py +# this third-level staging file overrides some definitions in staging_example.py # You may wish to alter it to agree with your local environment # # get most settings from staging_example.py (which in turn, imports from # settings.py) # flake8: noqa -from onadata.settings.common import * + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "", + "HOST": "127.0.0.1", } } DATABASE_ROUTERS = [] # turn off second database # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" DEBUG = True -INSTALLED_APPS += ('debug_toolbar', ) -REST_FRAMEWORK['DEFAULT_RENDERER_CLASSES'] += ( - 'onadata.libs.renderers.renderers.DebugToolbarRenderer', +INSTALLED_APPS += ("debug_toolbar",) +REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] += ( + "onadata.libs.renderers.renderers.DebugToolbarRenderer", ) diff --git a/onadata/settings/default_settings.py b/onadata/settings/default_settings.py index ae50178d52..16669f7741 100644 --- a/onadata/settings/default_settings.py +++ b/onadata/settings/default_settings.py @@ -1,25 +1,29 @@ +# -*- coding: utf-8 -*- +""" +Default settings module. +""" # this system uses structured settings.py as defined in # http://www.slideshare.net/jacobian/the-best-and-worst-of-django # -# this third-level staging file overrides some definitions in staging.py +# this third-level staging file overrides some definitions in staging_example.py # You may wish to alter it to agree with your local environment # # get most settings from staging_example.py (which in turn, imports from # settings.py) -from onadata.settings.staging_example import * # noqa +from onadata.settings.staging_example import * # noqa pylint: disable=W0401,W0614 -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "", + "HOST": "127.0.0.1", } } @@ -27,4 +31,4 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index 3c80d21c48..558b73635a 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -15,17 +15,17 @@ import subprocess import sys -from onadata.settings.common import * # noqa +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # # # now override the settings which came from staging # # # # DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': 'onadata', - 'HOST': 'db', - 'PORT': 5432 + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "onadata", + "HOST": "db", + "PORT": 5432, } } @@ -33,11 +33,11 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = '~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b' +SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b" -ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] +ALLOWED_HOSTS = ["127.0.0.1", "localhost"] -INTERNAL_IPS = ['127.0.0.1'] +INTERNAL_IPS = ["127.0.0.1"] DEBUG = True CORS_ORIGIN_ALLOW_ALL = True @@ -50,61 +50,58 @@ else: TESTING_MODE = False -CELERY_BROKER_URL = 'redis://queue:6379' -CELERY_RESULT_BACKEND = 'redis://queue:6379' +CELERY_BROKER_URL = "redis://queue:6379" +CELERY_RESULT_BACKEND = "redis://queue:6379" CELERY_TASK_ALWAYS_EAGER = True -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_CACHE_BACKEND = 'memory' +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_CACHE_BACKEND = "memory" CELERY_BROKER_CONNECTION_MAX_RETRIES = 2 CACHES = { - 'default': { - 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://queue:6379', - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient" - }, + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://queue:6379", + "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, } } NOTIFICATION_BACKENDS = { - 'mqtt': { - 'BACKEND': 'onadata.apps.messaging.backends.mqtt.MQTTBackend', - 'OPTIONS': { - 'HOST': 'notifications', - 'PORT': 1883, - 'QOS': 1, - 'RETAIN': False, - 'SECURE': False, - 'TOPIC_BASE': 'onadata' - } + "mqtt": { + "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend", + "OPTIONS": { + "HOST": "notifications", + "PORT": 1883, + "QOS": 1, + "RETAIN": False, + "SECURE": False, + "TOPIC_BASE": "onadata", + }, } } FULL_MESSAGE_PAYLOAD = True -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') # noqa + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # noqa subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have TASK_ALWAYS_EAGERY True and BROKER_URL as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" NOTIFICATION_BACKENDS = {} else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') # noqa + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") # noqa -ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' +ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" SUBMISSION_RETRIEVAL_THRESHOLD = 1000 CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD = 100000 - diff --git a/onadata/settings/drone_test.py b/onadata/settings/drone_test.py index bd14196e01..b611bab2cd 100644 --- a/onadata/settings/drone_test.py +++ b/onadata/settings/drone_test.py @@ -1,42 +1,49 @@ +# -*- coding:utf-8 -*- +""" +Example local_settings.py for use with DroneCI. +""" # flake8: noqa # this preset is used for automated testing of formhub # -from onadata.settings.common import * +import subprocess + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata_test', - 'USER': 'postgres', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata_test", + "USER": "postgres", + "PASSWORD": "", + "HOST": "127.0.0.1", } } -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "please replace this text" PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', - 'django.contrib.auth.hashers.SHA1PasswordHasher', + "django.contrib.auth.hashers.MD5PasswordHasher", + "django.contrib.auth.hashers.SHA1PasswordHasher", ) +DEBUG = True + if PRINT_EXCEPTION and DEBUG: - MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',) + MIDDLEWARE += ("utils.middleware.ExceptionLoggingMiddleware",) -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") diff --git a/onadata/settings/github_actions_test.py b/onadata/settings/github_actions_test.py index 7f298106e2..2346b71722 100644 --- a/onadata/settings/github_actions_test.py +++ b/onadata/settings/github_actions_test.py @@ -1,70 +1,70 @@ +# -*- coding: utf-8 -*- +""" +Django settings module or use on GitHub actions. +""" # flake8: noqa # this preset is used for automated testing of onadata from __future__ import absolute_import -from .common import * # nopep8 +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # database settings DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata', - 'USER': 'onadata', - 'PASSWORD': 'onadata', - 'HOST': 'localhost' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata", + "USER": "onadata", + "PASSWORD": "onadata", + "HOST": "localhost", } } SLAVE_DATABASES = [] -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' # nosec +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # nosec -JWT_SECRET_KEY = 'thesecretkey' # nosec -JWT_ALGORITHM = 'HS256' +JWT_SECRET_KEY = "thesecretkey" # nosec +JWT_ALGORITHM = "HS256" -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # subprocess.call(["rm", "-r", MEDIA_ROOT]) # nosec # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_BROKER_URL = "memory://" CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' # nosec - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" # nosec + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") -PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', -) +PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) DEBUG = False -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", ) -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' # nosec +ODK_TOKEN_FERNET_KEY = "ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=" # nosec OPENID_CONNECT_PROVIDERS = {} - diff --git a/onadata/settings/production_example.py b/onadata/settings/production_example.py index 5023d94d7f..ea57dcf820 100644 --- a/onadata/settings/production_example.py +++ b/onadata/settings/production_example.py @@ -1,5 +1,10 @@ +# -*- coding=utf-8 -*- +""" +Example local_settings.py used by the Dockerfile. +""" # flake8: noqa -from onadata.settings.common import * + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 # this setting file will not work on "runserver" -- it needs a server for # static files @@ -7,33 +12,31 @@ # override to set the actual location for the production static and media # directories -MEDIA_ROOT = '/var/formhub-media' +MEDIA_ROOT = "/var/formhub-media" STATIC_ROOT = "/srv/formhub-static" -STATICFILES_DIRS = ( - os.path.join(PROJECT_ROOT, "static"), -) +STATICFILES_DIRS = (os.path.join(PROJECT_ROOT, "static"),) ADMINS = ( # ('Your Name', 'your_email@example.com'), ) # your actual production settings go here...,. DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'formhub', - 'USER': 'formhub_prod', + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "formhub", + "USER": "formhub_prod", # the password must be stored in an environment variable - 'PASSWORD': os.environ['FORMHUB_PROD_PW'], + "PASSWORD": os.environ["FORMHUB_PROD_PW"], # the server name may be in env - 'HOST': os.environ.get("FORMHUB_DB_SERVER", 'dbserver.yourdomain.org') + "HOST": os.environ.get("FORMHUB_DB_SERVER", "dbserver.yourdomain.org"), }, - 'gis': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'phis', - 'USER': 'staff', + "gis": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "phis", + "USER": "staff", # the password must be stored in an environment variable - 'PASSWORD': os.environ['PHIS_PW'], - 'HOST': 'gisserver.yourdomain.org' - } + "PASSWORD": os.environ["PHIS_PW"], + "HOST": "gisserver.yourdomain.org", + }, } # Local time zone for this installation. Choices can be found here: @@ -43,36 +46,36 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'Africa/Lagos' +TIME_ZONE = "Africa/Lagos" -TOUCHFORMS_URL = 'http://localhost:9000/' +TOUCHFORMS_URL = "http://localhost:9000/" # Make this unique, and don't share it with anybody. -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfo6HOktYXB9y' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfo6HOktYXB9y" # Caching CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', + "default": { + "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "LOCATION": "127.0.0.1:11211", } } -MIDDLEWARE_CLASSES += ('django.middleware.cache.UpdateCacheMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.cache.FetchFromCacheMiddleware',) +MIDDLEWARE += ( + "django.middleware.cache.UpdateCacheMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.cache.FetchFromCacheMiddleware", +) CACHE_MIDDLEWARE_SECONDS = 3600 # 1 hour -CACHE_MIDDLEWARE_KEY_PREFIX = '' +CACHE_MIDDLEWARE_KEY_PREFIX = "" REST_SERVICES_TO_MODULES = { - 'google_sheets': 'google_export.services', + "google_sheets": "google_export.services", } REST_SERVICES_TO_SERIALIZERS = { - 'google_sheets': 'google_export.serializers.GoogleSheetsSerializer' + "google_sheets": "google_export.serializers.GoogleSheetsSerializer" } -CUSTOM_MAIN_URLS = { - 'google_export.urls' -} +CUSTOM_MAIN_URLS = {"google_export.urls"} diff --git a/onadata/settings/staging_example.py b/onadata/settings/staging_example.py index 630a9fab03..9436355644 100644 --- a/onadata/settings/staging_example.py +++ b/onadata/settings/staging_example.py @@ -1,45 +1,50 @@ -# flake8: noqa -from onadata.settings.common import * +# -*- coding: utf-8 -*- +""" +Example staging module. +""" +import os +import subprocess +import sys + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -TEMPLATE_STRING_IF_INVALID = '' +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG # noqa +TEMPLATE_STRING_IF_INVALID = "" # see: http://docs.djangoproject.com/en/dev/ref/settings/#databases # postgres DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'formhub_dev', - 'USER': 'formhub_dev', - 'PASSWORD': '12345678', - 'HOST': 'localhost' + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "formhub_dev", + "USER": "formhub_dev", + "PASSWORD": "12345678", + "HOST": "localhost", }, } # TIME_ZONE = 'UTC' -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#srgcpj%dd*sisfo6HOktYXB9y' +SECRET_KEY = "please replace this text" -TESTING_MODE = False -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') - subprocess.call(["rm", "-r", MEDIA_ROOT]) + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") # noqa + subprocess.call(["rm", "-r", MEDIA_ROOT]) # noqa # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") # noqa -if PRINT_EXCEPTION and DEBUG: - MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',) +if PRINT_EXCEPTION and DEBUG: # noqa + MIDDLEWARE += ("utils.middleware.ExceptionLoggingMiddleware",) # noqa diff --git a/onadata/settings/travis_test.py b/onadata/settings/travis_test.py index 4bda303111..98f75cb8e3 100644 --- a/onadata/settings/travis_test.py +++ b/onadata/settings/travis_test.py @@ -1,69 +1,73 @@ +# -*- coding: utf-8 -*- +""" +Django settings module or use on GitHub actions. +""" # flake8: noqa -# this preset is used for automated testing of formhub +# this preset is used for automated testing of onadata from __future__ import absolute_import -from .common import * # nopep8 +import subprocess + +from onadata.settings.common import * # noqa pylint: disable=W0401,W0614 + # database settings DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'onadata_test', - 'USER': 'postgres', - 'PASSWORD': '', - 'HOST': '127.0.0.1' + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "onadata_test", + "USER": "postgres", + "PASSWORD": "", + "HOST": "127.0.0.1", } } SLAVE_DATABASES = [] -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j' +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" -JWT_SECRET_KEY = 'thesecretkey' -JWT_ALGORITHM = 'HS256' +JWT_SECRET_KEY = "thesecretkey" +JWT_ALGORITHM = "HS256" -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True -else: - TESTING_MODE = False +# This trick works only when we run tests from the command line. +TESTING_MODE = len(sys.argv) >= 2 and ( + sys.argv[1] == "test" or sys.argv[1] == "test_all" +) if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "test_media/") subprocess.call(["rm", "-r", MEDIA_ROOT]) # need to have CELERY_TASK_ALWAYS_EAGER True and BROKER_BACKEND as memory # to run tasks immediately while testing CELERY_BROKER_URL = "memory://" CELERY_TASK_ALWAYS_EAGER = True - CELERY_RESULT_BACKEND = 'cache' - CELERY_CACHE_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' - ENKETO_PROTOCOL = 'https' - ENKETO_URL = 'https://enketo.ona.io/' - ENKETO_API_ALL_SURVEY_LINKS_PATH = '/api_v2/survey/all' - ENKETO_API_INSTANCE_PATH = '/api_v1/instance' - ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' + CELERY_RESULT_BACKEND = "cache" + CELERY_CACHE_BACKEND = "memory" + ENKETO_API_TOKEN = "abc" + ENKETO_PROTOCOL = "https" + ENKETO_URL = "https://enketo.ona.io/" + ENKETO_API_ALL_SURVEY_LINKS_PATH = "/api_v2/survey/all" + ENKETO_API_INSTANCE_PATH = "/api_v1/instance" + ENKETO_SINGLE_SUBMIT_PATH = "/api/v2/survey/single/once" ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') + MEDIA_ROOT = os.path.join(PROJECT_ROOT, "media/") -PASSWORD_HASHERS = ( - 'django.contrib.auth.hashers.MD5PasswordHasher', -) +PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) DEBUG = False -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', +TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG +MIDDLEWARE = ( + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", # 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware", ) -VERIFIED_KEY_TEXT = 'ALREADY_ACTIVATED' +VERIFIED_KEY_TEXT = "ALREADY_ACTIVATED" -ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' +ODK_TOKEN_FERNET_KEY = "ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=" OPENID_CONNECT_PROVIDERS = {} From 752fdb3c70193981c1fdeb45e4967c21d7fc67ef Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Apr 2022 22:50:14 +0300 Subject: [PATCH 048/234] Cleanup --- onadata/apps/api/models/temp_token.py | 11 ++- .../viewsets/test_user_profile_viewset.py | 78 +++++++++++-------- onadata/apps/logger/models/data_view.py | 59 +++++++------- 3 files changed, 81 insertions(+), 67 deletions(-) diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index ef782ad901..c9ded3c28e 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -5,11 +5,9 @@ import binascii import os -from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models -AUTH_USER_MODEL = getattr(settings, "AUTH_USER_MODEL", "auth.User") - class TempToken(models.Model): @@ -19,7 +17,7 @@ class TempToken(models.Model): key = models.CharField(max_length=40, primary_key=True) user = models.OneToOneField( - AUTH_USER_MODEL, related_name="_user", on_delete=models.CASCADE + get_user_model(), related_name="_user", on_delete=models.CASCADE ) created = models.DateTimeField(auto_now_add=True) @@ -29,9 +27,10 @@ class Meta: def save(self, *args, **kwargs): if not self.key: self.key = self.generate_key() - return super(TempToken, self).save(*args, **kwargs) + return super().save(*args, **kwargs) - def generate_key(self): + def generate_key(self): # pylint: disable=no-self-use + """Generates a token key.""" return binascii.hexlify(os.urandom(20)).decode() def __str__(self): diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index b525b3f523..265b201d7b 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -1,10 +1,14 @@ +# -*- coding: utf-8 -*- +""" +Tests the UserProfileViewSet. +""" +# pylint: disable=too-many-lines import datetime import json import os -from builtins import str from six.moves.urllib.parse import urlparse, parse_qs -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.db.models import signals from django.test.utils import override_settings @@ -28,6 +32,9 @@ from onadata.libs.serializers.user_profile_serializer import _get_first_last_names +User = get_user_model() + + def _profile_data(): return { "username": "deno", @@ -46,9 +53,12 @@ def _profile_data(): } +# pylint: disable=attribute-defined-outside-init,too-many-public-methods +# pylint: disable=invalid-name,missing-class-docstring,missing-function-docstring, +# pylint: disable=consider-using-f-string class TestUserProfileViewSet(TestAbstractViewSet): def setUp(self): - super(self.__class__, self).setUp() + super().setUp() self.view = UserProfileViewSet.as_view( { "get": "list", @@ -62,7 +72,7 @@ def tearDown(self): """ Specific to clear cache between tests """ - super(TestUserProfileViewSet, self).tearDown() + super().tearDown() cache.clear() def test_profiles_list(self): @@ -302,14 +312,14 @@ def test_return_204_if_email_verification_variables_are_not_set(self): _data = {"verification_key": rp.activation_key} request = self.factory.get("/", data=_data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) data = {"username": data.get("username")} user = User.objects.get(username=data.get("username")) extra = {"HTTP_AUTHORIZATION": "Token %s" % user.auth_token} request = self.factory.post("/", data=data, **extra) response = view(request) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) @override_settings(ENABLE_EMAIL_VERIFICATION=True) def test_verification_key_is_valid(self): @@ -321,11 +331,11 @@ def test_verification_key_is_valid(self): _data = {"verification_key": rp.activation_key} request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertIn("is_email_verified", response.data) self.assertIn("username", response.data) self.assertTrue(response.data.get("is_email_verified")) - self.assertEquals(response.data.get("username"), data.get("username")) + self.assertEqual(response.data.get("username"), data.get("username")) up = UserProfile.objects.get(user__username=data.get("username")) self.assertIn("is_email_verified", up.metadata) @@ -345,14 +355,14 @@ def test_verification_key_is_valid_with_redirect_url_set(self): request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 302) + self.assertEqual(response.status_code, 302) self.assertIn("is_email_verified", response.url) self.assertIn("username", response.url) string_query_params = urlparse(response.url).query dict_query_params = parse_qs(string_query_params) - self.assertEquals(dict_query_params.get("is_email_verified"), ["True"]) - self.assertEquals(dict_query_params.get("username"), [data.get("username")]) + self.assertEqual(dict_query_params.get("is_email_verified"), ["True"]) + self.assertEqual(dict_query_params.get("username"), [data.get("username")]) up = UserProfile.objects.get(user__username=data.get("username")) self.assertIn("is_email_verified", up.metadata) @@ -377,8 +387,8 @@ def test_sending_verification_email_succeeds(self, mock_send_verification_email) request = self.factory.post("/", data=data, **extra) response = view(request) self.assertTrue(mock_send_verification_email.called) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, "Verification email has been sent") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, "Verification email has been sent") user = User.objects.get(username=data.get("username")) self.assertFalse(user.profile.metadata.get("is_email_verified")) @@ -395,7 +405,7 @@ def test_sending_verification_email_fails(self): # different from username in post details request = self.factory.post("/", data={"username": "None"}, **self.extra) response = view(request) - self.assertEquals(response.status_code, 403) + self.assertEqual(response.status_code, 403) @override_settings(REQUIRE_ODK_AUTHENTICATION=True) def test_profile_require_auth(self): @@ -578,7 +588,7 @@ def test_partial_updates(self): def test_partial_updates_empty_metadata(self): profile = UserProfile.objects.get(user=self.user) - profile.metadata = dict() + profile.metadata = {} profile.save() metadata = {"zebra": {"key1": "value1", "key2": "value2"}} json_metadata = json.dumps(metadata) @@ -1103,16 +1113,20 @@ def test_profile_create_fails_with_long_first_and_last_names(self): self.assertEqual(response.status_code, 400) @all_requests - def grant_perms_form_builder(self, url, request): + def grant_perms_form_builder( + self, url, request + ): # pylint: disable=no-self-use,unused-argument assert "Authorization" in request.headers assert request.headers.get("Authorization").startswith("Token") response = requests.Response() response.status_code = 201 - response._content = { - "detail": "Successfully granted default model level perms to" " user." - } + setattr( + response, + "_content", + {"detail": "Successfully granted default model level perms to" " user."}, + ) return response def test_create_user_with_given_name(self): @@ -1135,13 +1149,13 @@ def test_get_monthly_submissions(self): request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {"private": count1}) + self.assertEqual(response.data, {"private": count1}) # publish another form, make submission and make it public self._publish_form_with_hxl_support() - self.assertEquals(self.xform.id_string, "hxl_example") + self.assertEqual(self.xform.id_string, "hxl_example") count2 = ( Instance.objects.filter(xform=self.xform) .filter(date_created__year=datetime.datetime.now().year) @@ -1154,8 +1168,8 @@ def test_get_monthly_submissions(self): self.xform.save() request = self.factory.get("/", **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data, {"private": count1, "public": count2}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, {"private": count1, "public": count2}) def test_get_monthly_submissions_with_year_and_month_params(self): """ @@ -1188,9 +1202,9 @@ def test_get_monthly_submissions_with_year_and_month_params(self): data = {"month": 2, "year": 2013} request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {"private": count}) + self.assertEqual(response.data, {"private": count}) def test_monthly_submissions_with_month_param(self): """ @@ -1214,9 +1228,9 @@ def test_monthly_submissions_with_month_param(self): data = {"month": month} request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {"private": count}) + self.assertEqual(response.data, {"private": count}) def test_monthly_submissions_with_year_param(self): """ @@ -1252,9 +1266,9 @@ def test_monthly_submissions_with_year_param(self): data = {"year": 2013} request = self.factory.get("/", data=data, **self.extra) response = view(request, user=self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertFalse(self.xform.shared) - self.assertEquals(response.data, {"private": count}) + self.assertEqual(response.data, {"private": count}) @override_settings(ENABLE_EMAIL_VERIFICATION=True) @patch("onadata.apps.api.viewsets.user_profile_viewset.RegistrationProfile") @@ -1275,11 +1289,11 @@ def test_reads_from_master(self, mock_rp_class): ).get.side_effect = [RegistrationProfile.DoesNotExist, rp] request = self.factory.get("/", data=_data) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertIn("is_email_verified", response.data) self.assertIn("username", response.data) self.assertTrue(response.data.get("is_email_verified")) - self.assertEquals(response.data.get("username"), data.get("username")) + self.assertEqual(response.data.get("username"), data.get("username")) self.assertEqual( mock_rp_class.objects.select_related( "user", "user__profile" diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index a6a02d62e3..925c31cb9b 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -15,12 +15,12 @@ from django.utils.translation import ugettext as _ from onadata.apps.viewer.parsed_instance_tools import get_where_clause -from onadata.libs.models.sorting import ( +from onadata.libs.models.sorting import ( # noqa pylint: disable=unused-import json_order_by, json_order_by_params, sort_from_mongo_sort_str, ) -from onadata.libs.utils.cache_tools import ( +from onadata.libs.utils.cache_tools import ( # noqa pylint: disable=unused-import DATAVIEW_COUNT, DATAVIEW_LAST_SUBMISSION_TIME, XFORM_LINKED_DATAVIEWS, @@ -62,9 +62,9 @@ def get_name_from_survey_element(element): def append_where_list(comp, t_list, json_str): if comp in ["=", ">", "<", ">=", "<="]: - t_list.append("{} {} %s".format(json_str, comp)) + t_list.append(f"{json_str} {comp} %s") elif comp in ["<>", "!="]: - t_list.append("{} <> %s".format(json_str)) + t_list.append("{json_str} <> %s") return t_list @@ -138,7 +138,7 @@ def has_geo_columnn_n_data(self): # Get the form geo xpaths xform = self.xform - geo_xpaths = xform.geopoint_xpaths() # pylint: disable=E1101 + geo_xpaths = xform.geopoint_xpaths() set_geom = set(geo_xpaths) set_columns = set(self.columns) @@ -152,10 +152,9 @@ def has_geo_columnn_n_data(self): def save(self, *args, **kwargs): self.instances_with_geopoints = self.has_geo_columnn_n_data() - return super(DataView, self).save(*args, **kwargs) + return super().save(*args, **kwargs) def _get_known_type(self, type_str): - # pylint: disable=E1101 return [ get_name_from_survey_element(e) for e in self.xform.get_survey_elements_of_type(type_str) @@ -191,15 +190,14 @@ def has_instance(self, instance): sql += ( " WHERE xform_id = %s AND id = %s" + sql_where + " AND deleted_at IS NULL" ) - # pylint: disable=E1101 params = [self.xform.pk, instance.id] + where_params cursor.execute(sql, [text(i) for i in params]) - + records = None for row in cursor.fetchall(): records = row[0] - return True if records else False + return records is not None def soft_delete(self, user=None): """ @@ -218,13 +216,16 @@ def soft_delete(self, user=None): self.save(update_fields=update_fields) @classmethod - def _get_where_clause( + def _get_where_clause( # pylint: disable=too-many-locals cls, data_view, - form_integer_fields=[], - form_date_fields=[], - form_decimal_fields=[], + form_integer_fields=None, + form_date_fields=None, + form_decimal_fields=None, ): + form_integer_fields = [] if form_integer_fields is None else form_integer_fields + form_date_fields = [] if form_date_fields is None else form_date_fields + form_decimal_fields = [] if form_decimal_fields is None else form_decimal_fields known_integers = ["_id"] + form_integer_fields known_dates = ["_submission_time"] + form_date_fields known_decimals = form_decimal_fields @@ -265,14 +266,15 @@ def _get_where_clause( return where, where_params @classmethod - def query_iterator(cls, sql, fields=None, params=[], count=False): - + def query_iterator(cls, sql, fields=None, params=None, count=False): def parse_json(data): try: return json.loads(data) except ValueError: return data + params = [] if params is None else params + cursor = connection.cursor() sql_params = tuple(i if isinstance(i, tuple) else text(i) for i in params) @@ -300,6 +302,7 @@ def parse_json(data): for row in cursor.fetchall(): yield dict(zip(fields, [parse_json(row[0]).get(f) for f in fields])) + # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @classmethod def generate_query_string( cls, @@ -366,7 +369,7 @@ def generate_query_string( if sort is not None: sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) - sql = "{} {}".format(sql, json_order_by(sort)) + sql = "{sql} {json_order_by(sort)}" params = params + json_order_by_params(sort) elif last_submission_time is False: @@ -390,7 +393,7 @@ def generate_query_string( ) @classmethod - def query_data( + def query_data( # pylint: disable=too-many-arguments cls, data_view, start_index=None, @@ -401,6 +404,7 @@ def query_data( sort=None, filter_query=None, ): + """Returns a list of records for the view based on the parameters passed in.""" (sql, columns, params) = cls.generate_query_string( data_view, @@ -413,27 +417,24 @@ def query_data( ) try: - records = [ - record - for record in DataView.query_iterator(sql, columns, params, count) - ] + records = list(DataView.query_iterator(sql, columns, params, count)) except Exception as e: return {"error": _(text(e))} return records -def clear_cache(sender, instance, **kwargs): +def clear_cache(sender, instance, **kwargs): # pylint: disable=unused-argument """Post delete handler for clearing the dataview cache.""" - safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + safe_delete("{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") -def clear_dataview_cache(sender, instance, **kwargs): +def clear_dataview_cache(sender, instance, **kwargs): # pylint: disable=unused-argument """Post Save handler for clearing dataview cache on serialized fields.""" - safe_delete("{}{}".format(PROJ_OWNER_CACHE, instance.project.pk)) - safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform.pk)) - safe_delete("{}{}".format(DATAVIEW_LAST_SUBMISSION_TIME, instance.xform.pk)) - safe_delete("{}{}".format(XFORM_LINKED_DATAVIEWS, instance.xform.pk)) + safe_delete("{PROJ_OWNER_CACHE}{instance.project.pk}") + safe_delete("{DATAVIEW_COUNT}{instance.xform.pk}") + safe_delete("{DATAVIEW_LAST_SUBMISSION_TIME}{instance.xform.pk}") + safe_delete("{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") post_save.connect(clear_dataview_cache, sender=DataView, dispatch_uid="clear_cache") From 8f11a5119ff17f679813bcb693f9f9edd496dcbb Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 10:25:46 +0300 Subject: [PATCH 049/234] Add lxml apt deps to CI Signed-off-by: Kipchirchir Sigei --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcc2e60ecc..4ed5223ec4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - name: Install APT requirements run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev + sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev sudo rm -rf /var/lib/apt/lists/* - name: Install Pip requirements From 23e3d890e0e123f6679c7a1d9de8a875804ff281 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 10:48:36 +0300 Subject: [PATCH 050/234] Cleanup Signed-off-by: Kipchirchir Sigei --- onadata/apps/api/viewsets/xform_viewset.py | 2 +- onadata/apps/viewer/models/data_dictionary.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index b373fe1e38..cbe33ddde3 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -418,7 +418,7 @@ def create_async(self, request, *args, **kwargs): @never_cache def form(self, request, format="json", **kwargs): form = self.get_object() - if format not in ["json", "xml", "xls", "csv"]: + if format not in ["json", "xml", "xls", "xlsx", "csv"]: return HttpResponseBadRequest( "400 BAD REQUEST", content_type="application/json", status=400 ) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index dae0466c76..cb5c3c9c99 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -8,7 +8,6 @@ import unicodecsv as csv import openpyxl from builtins import str as text -from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone @@ -52,10 +51,10 @@ def process_xlsform(xls, default_name): file_object = xls if xls.name.endswith("csv"): file_object = None - if not get_storage_class()().exists(xls.path): + if not isinstance(xls.name, InMemoryUploadedFile): file_object = StringIO(xls.read().decode("utf-8")) try: - return parse_file_to_json(xls.path, file_object=file_object) + return parse_file_to_json(xls.name, file_object=file_object) except csv.Error as e: if is_newline_error(e): xls.seek(0) From 1d3f36336188f5db682061a06e0bc26813acdd46 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 12:52:41 +0300 Subject: [PATCH 051/234] Fix deformed f-strings Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/data_view.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 925c31cb9b..b5a41efdff 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -62,9 +62,9 @@ def get_name_from_survey_element(element): def append_where_list(comp, t_list, json_str): if comp in ["=", ">", "<", ">=", "<="]: - t_list.append(f"{json_str} {comp} %s") + t_list.append(f"{json_str} {comp}" + " %s") elif comp in ["<>", "!="]: - t_list.append("{json_str} <> %s") + t_list.append(f"{json_str} <>" + " %s") return t_list @@ -369,7 +369,7 @@ def generate_query_string( if sort is not None: sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) - sql = "{sql} {json_order_by(sort)}" + sql = f"{sql} {json_order_by(sort)}" params = params + json_order_by_params(sort) elif last_submission_time is False: @@ -426,15 +426,15 @@ def query_data( # pylint: disable=too-many-arguments def clear_cache(sender, instance, **kwargs): # pylint: disable=unused-argument """Post delete handler for clearing the dataview cache.""" - safe_delete("{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") + safe_delete(f"{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") def clear_dataview_cache(sender, instance, **kwargs): # pylint: disable=unused-argument """Post Save handler for clearing dataview cache on serialized fields.""" - safe_delete("{PROJ_OWNER_CACHE}{instance.project.pk}") - safe_delete("{DATAVIEW_COUNT}{instance.xform.pk}") - safe_delete("{DATAVIEW_LAST_SUBMISSION_TIME}{instance.xform.pk}") - safe_delete("{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") + safe_delete(f"{PROJ_OWNER_CACHE}{instance.project.pk}") + safe_delete(f"{DATAVIEW_COUNT}{instance.xform.pk}") + safe_delete(f"{DATAVIEW_LAST_SUBMISSION_TIME}{instance.xform.pk}") + safe_delete(f"{XFORM_LINKED_DATAVIEWS}{instance.xform.pk}") post_save.connect(clear_dataview_cache, sender=DataView, dispatch_uid="clear_cache") From 69621ca2785ec82de14bd134cd52ba6c1dae2a7f Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 15:00:48 +0300 Subject: [PATCH 052/234] Upgrade to django 3 on ses & s3 requirements Signed-off-by: Kipchirchir Sigei --- requirements/dev.pip | 2 +- requirements/s3.pip | 2 +- requirements/ses.pip | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev.pip b/requirements/dev.pip index 4c0ae4b8b9..ce5fbd0853 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -91,7 +91,7 @@ deprecated==1.2.12 # via onadata dict2xml==1.7.0 # via onadata -django==2.2.23 +django==3.2.13 # via # django-cors-headers # django-debug-toolbar diff --git a/requirements/s3.pip b/requirements/s3.pip index a5b4dd4de7..f27f1c9c0c 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -12,7 +12,7 @@ botocore==1.20.74 # s3transfer django-storages==1.11.1 # via -r requirements/s3.in -django==2.2.23 +django==3.2.13 # via # -r requirements/s3.in # django-storages diff --git a/requirements/ses.pip b/requirements/ses.pip index ea7f904311..5c2739220e 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -14,7 +14,7 @@ botocore==1.20.74 # s3transfer django-ses==2.0.0 # via -r requirements/ses.in -django==2.2.23 +django==3.2.13 # via # -r requirements/ses.in # django-ses From 93102a412b674b5b742c531b11cd24d3fa7265a5 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 18:46:03 +0300 Subject: [PATCH 053/234] covert survey to json string Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/xform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 659091dc43..2d818f2fb6 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -332,7 +332,7 @@ def get_survey(self): if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() - self._survey = builder.create_survey_element_from_json(self.json) + self._survey = builder.create_survey_element_from_json(json.dumps(self.json)) except ValueError: xml = b(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) From 323bb63560746e19e82dcad745ed6c2615a2416d Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 28 Apr 2022 18:46:03 +0300 Subject: [PATCH 054/234] covert survey to json string Signed-off-by: Kipchirchir Sigei --- onadata/libs/utils/api_export_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 7becf8c34a..30d4f2f09a 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -572,7 +572,7 @@ def response_for_format(data, format=None): # pylint: disable=W0622 """ if format == "xml": formatted_data = data.xml - elif format == "xls": + elif format == "xls" or format == "xlsx": if not data.xls or not data.xls.storage.exists(data.xls.name): raise Http404() From c74117085166d808f7ca9db957c364597dffc553 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 28 Apr 2022 19:47:56 +0300 Subject: [PATCH 055/234] Update OSM XML, order changed. --- onadata/apps/api/tests/fixtures/osm/combined.osm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/fixtures/osm/combined.osm b/onadata/apps/api/tests/fixtures/osm/combined.osm index 3457809426..20ed3d01de 100644 --- a/onadata/apps/api/tests/fixtures/osm/combined.osm +++ b/onadata/apps/api/tests/fixtures/osm/combined.osm @@ -1,2 +1,2 @@ - + From 9a9314b79f9944c885d4fabbc4e92a1cfa37959c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 05:41:06 +0300 Subject: [PATCH 056/234] Use builder.create_survey_element_from_(dict|json) accordingly. --- onadata/apps/logger/models/xform.py | 5 +++- onadata/apps/main/tests/test_http_auth.py | 30 ++++++++++++----------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 2d818f2fb6..0fdcd4de98 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -332,7 +332,10 @@ def get_survey(self): if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() - self._survey = builder.create_survey_element_from_json(json.dumps(self.json)) + if isinstance(self.json, str): + self._survey = builder.create_survey_element_from_json(self.json) + if isinstance(self.json, dict): + self._survey = builder.create_survey_element_from_dict(self.json) except ValueError: xml = b(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) diff --git a/onadata/apps/main/tests/test_http_auth.py b/onadata/apps/main/tests/test_http_auth.py index 7d747cef6f..ca7e7f440b 100644 --- a/onadata/apps/main/tests/test_http_auth.py +++ b/onadata/apps/main/tests/test_http_auth.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +"""Test basic authentication""" from django.urls import reverse from onadata.apps.main import views @@ -5,41 +7,41 @@ class TestBasicHttpAuthentication(TestBase): + """Test basic authentication""" def setUp(self): TestBase.setUp(self) - self._create_user_and_login(username='bob', password='bob') + self._create_user_and_login(username="bob", password="bob") self._publish_transportation_form() - self.api_url = reverse(views.api, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + self.api_url = reverse( + views.api, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) self._logout() def test_http_auth(self): response = self.client.get(self.api_url) self.assertEqual(response.status_code, 403) # headers with invalid user/pass - response = self.client.get(self.api_url, - **self._set_auth_headers('x', 'y')) + response = self.client.get(self.api_url, **self._set_auth_headers("x", "y")) self.assertEqual(response.status_code, 403) # headers with valid user/pass - response = self.client.get(self.api_url, - **self._set_auth_headers('bob', 'bob')) - self.assertEqual(response.status_code, 200) + response = self.client.get(self.api_url, **self._set_auth_headers("bob", "bob")) + self.assertEqual(response.status_code, 200, response.content) # Set user email self.user.email = "bob@testing_pros.com" self.user.save() # headers with valid email/pass response = self.client.get( - self.api_url, **self._set_auth_headers(self.user.email, 'bob')) - self.assertEqual(response.status_code, 200) + self.api_url, **self._set_auth_headers(self.user.email, "bob") + ) + self.assertEqual(response.status_code, 200, response.content) def test_http_auth_shared_data(self): self.xform.shared_data = True self.xform.save() response = self.anon.get(self.api_url) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, response.content) response = self.client.get(self.api_url) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, response.content) From 879240d16eaca2b2d3189b8087d3503d687917d5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 09:05:33 +0300 Subject: [PATCH 057/234] Cleanup --- onadata/apps/logger/models/xform.py | 209 +++++++++++--------- onadata/apps/logger/signals.py | 42 +++- onadata/apps/viewer/models/column_rename.py | 11 +- 3 files changed, 162 insertions(+), 100 deletions(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 0fdcd4de98..5869705246 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +""" +The XForm model +""" +# pylint: disable=too-many-lines import json import os import re @@ -8,15 +13,16 @@ import pytz from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.contenttypes.fields import GenericRelation from django.core.exceptions import ObjectDoesNotExist from django.core.cache import cache from django.db import models, transaction from django.db.models import Sum -from django.db.models.signals import post_delete, post_save, pre_save +from django.db.models.signals import post_delete, pre_save from django.urls import reverse from django.utils import timezone +from django.utils.html import conditional_escape from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy from six import iteritems @@ -28,10 +34,8 @@ from taggit.managers import TaggableManager from onadata.apps.logger.xform_instance_parser import XLSFormError, clean_and_parse_xml -from django.utils.html import conditional_escape from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.cache_tools import ( - IS_ORG, PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, @@ -69,6 +73,8 @@ XFORM_TITLE_LENGTH = 255 title_pattern = re.compile(r"(.*?)") +User = get_user_model() + def cmp(x, y): return (x > y) - (x < y) @@ -82,19 +88,23 @@ def upload_to(instance, filename): return os.path.join(instance.user.username, "xls", os.path.split(filename)[1]) -def contains_xml_invalid_char(text, invalids=["&", ">", "<"]): +def contains_xml_invalid_char(text, invalids=None): """Check whether 'text' contains ANY invalid xml chars""" + invalids = ["&", ">", "<"] if invalids is None else invalids return 1 in [c in text for c in invalids] -class DictOrganizer(object): +class DictOrganizer: def set_dict_iterator(self, dict_iterator): self._dict_iterator = dict_iterator # Every section will get its own table # I need to think of an easy way to flatten out a dictionary # parent name, index, table name, data - def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_index): + # pylint: disable=too-many-arguments + def _build_obs_from_dict( + self, dict_item, obs, table_name, parent_table_name, parent_index + ): if table_name not in obs: obs[table_name] = [] this_index = len(obs[table_name]) @@ -104,17 +114,17 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_ind "_parent_index": parent_index, } ) - for (k, v) in iteritems(d): + for (k, v) in iteritems(dict_item): if isinstance(v, dict) and isinstance(v, list): if k in obs[table_name][-1]: raise AssertionError() obs[table_name][-1][k] = v obs[table_name][-1]["_index"] = this_index - for (k, v) in iteritems(d): + for (k, v) in iteritems(dict_item): if isinstance(v, dict): kwargs = { - "d": v, + "dict_item": v, "obs": obs, "table_name": k, "parent_table_name": table_name, @@ -124,7 +134,7 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_ind elif isinstance(v, list): for item in v: kwargs = { - "d": item, + "dict_item": item, "obs": obs, "table_name": k, "parent_table_name": table_name, @@ -133,12 +143,12 @@ def _build_obs_from_dict(self, d, obs, table_name, parent_table_name, parent_ind self._build_obs_from_dict(**kwargs) return obs - def get_observation_from_dict(self, d): - if len(list(d)) != 1: + def get_observation_from_dict(self, item): + if len(list(item)) != 1: raise AssertionError() - root_name = list(d)[0] + root_name = list(item)[0] kwargs = { - "d": d[root_name], + "dict_item": item[root_name], "obs": {}, "table_name": root_name, "parent_table_name": "", @@ -183,29 +193,32 @@ def check_version_set(survey): return survey -def _expand_select_all_that_apply(d, key, e): - if e and e.bind.get("type") == "string" and e.type == MULTIPLE_SELECT_TYPE: - options_selected = d[key].split() - for child in e.children: +def _expand_select_all_that_apply(item, key, elem): + """Split's a select multiple into individual keys""" + if elem and elem.bind.get("type") == "string" and elem.type == MULTIPLE_SELECT_TYPE: + options_selected = item[key].split() + for child in elem.children: new_key = child.get_abbreviated_xpath() - d[new_key] = child.name in options_selected + item[new_key] = child.name in options_selected - del d[key] + del item[key] -class XFormMixin(object): +class XFormMixin: + """XForm mixin class - adds helper functions.""" GEODATA_SUFFIXES = ["latitude", "longitude", "altitude", "precision"] PREFIX_NAME_REGEX = re.compile(r"(?P.+/)(?P[^/]+)$") + # pylint: disable=too-many-locals def _set_uuid_in_xml(self, file_name=None): """ Add bind to automatically set UUID node in XML. """ if not file_name: file_name = self.file_name() - file_name, file_ext = os.path.splitext(file_name) + file_name, _file_ext = os.path.splitext(file_name) doc = clean_and_parse_xml(self.xml) model_nodes = doc.getElementsByTagName("model") @@ -238,7 +251,7 @@ def _set_uuid_in_xml(self, file_name=None): ] if len(survey_nodes) != 1: - raise Exception("Multiple survey nodes with the id '%s'" % self.id_string) + raise Exception("Multiple survey nodes with the id '{self.id_string}'") survey_node = survey_nodes[0] formhub_nodes = [ @@ -249,7 +262,7 @@ def _set_uuid_in_xml(self, file_name=None): if len(formhub_nodes) > 1: raise Exception("Multiple formhub nodes within main instance node") - elif len(formhub_nodes) == 1: + if len(formhub_nodes) == 1: formhub_node = formhub_nodes[0] else: formhub_node = survey_node.insertBefore( @@ -268,10 +281,10 @@ def _set_uuid_in_xml(self, file_name=None): # append the calculate bind node calculate_node = doc.createElement("bind") calculate_node.setAttribute( - "nodeset", "/%s/formhub/uuid" % survey_node.tagName + "nodeset", f"/{survey_node.tagName}/formhub/uuid" ) calculate_node.setAttribute("type", "string") - calculate_node.setAttribute("calculate", "'%s'" % self.uuid) + calculate_node.setAttribute("calculate", f"'{self.uuid}'") model_node.appendChild(calculate_node) self.xml = doc.toprettyxml(indent=" ", encoding="utf-8") @@ -322,7 +335,7 @@ def get_unique_id_string(self, id_string, count=0): a = id_string.split("_") id_string = "_".join(a[:-1]) count += 1 - id_string = "{}_{}".format(id_string, count) + id_string = f"{id_string}_{count}" return self.get_unique_id_string(id_string, count) @@ -373,7 +386,8 @@ def get_child_elements(self, name_or_xpath, split_select_multiples=True): if split_select_multiples: GROUP_AND_SELECT_MULTIPLES += ["select all that apply"] - def flatten(elem, items=[]): + def flatten(elem, items=None): + items = [] if items is None else items results = [] if elem: xpath = elem.get_abbreviated_xpath() @@ -422,7 +436,7 @@ def get_field_name_xpaths_only(self): return [ elem.get_abbreviated_xpath() for elem in self.survey_elements - if elem.type != "" and elem.type != "survey" + if elem.type not in ("", "survey") ] def geopoint_xpaths(self): @@ -457,7 +471,7 @@ def xpaths(self, prefix="", survey_element=None, result=None, repeat_iterations= indices = ( [""] if not isinstance(survey_element, RepeatingSection) - else ["[%d]" % (i + 1) for i in range(repeat_iterations)] + else [f"[{(i + 1)}]" for i in range(repeat_iterations)] ) for i in indices: for e in survey_element.children: @@ -539,12 +553,15 @@ def shorten(xpath): return header_list def get_keys(self): + """Return all XForm headers.""" + def remove_first_index(xpath): return re.sub(r"\[1\]", "", xpath) return [remove_first_index(header) for header in self.get_headers()] def get_element(self, abbreviated_xpath): + """Returns an XML element""" if not hasattr(self, "_survey_elements"): self._survey_elements = {} for e in self.get_survey_elements(): @@ -557,6 +574,7 @@ def remove_all_indices(xpath): return self._survey_elements.get(clean_xpath) def get_default_language(self): + """Returns the default language""" if not hasattr(self, "_default_language"): self._default_language = self.survey.to_json_dict().get("default_language") @@ -565,6 +583,7 @@ def get_default_language(self): default_language = property(get_default_language) def get_language(self, languages, language_index=0): + """Returns the language at the given index.""" language = None if isinstance(languages, list) and len(languages): if self.default_language in languages: @@ -575,6 +594,7 @@ def get_language(self, languages, language_index=0): return language def get_label(self, abbreviated_xpath, elem=None, language=None): + """Returns the label of given xpath.""" elem = self.get_element(abbreviated_xpath) if elem is None else elem if elem: @@ -588,8 +608,10 @@ def get_label(self, abbreviated_xpath, elem=None, language=None): label = label[language] if language else "" return label + return None def get_xpath_cmp(self): + """Compare two xpaths""" if not hasattr(self, "_xpaths"): self._xpaths = [e.get_abbreviated_xpath() for e in self.survey_elements] @@ -602,9 +624,9 @@ def xpath_cmp(x, y): return cmp(x, y) if new_x not in self._xpaths and new_y not in self._xpaths: return 0 - elif new_x not in self._xpaths: + if new_x not in self._xpaths: return 1 - elif new_y not in self._xpaths: + if new_y not in self._xpaths: return -1 return cmp(self._xpaths.index(new_x), self._xpaths.index(new_y)) @@ -627,6 +649,7 @@ def get_variable_name(self, abbreviated_xpath): header = self._headers[i] if not hasattr(self, "_variable_names"): + # pylint: disable=import-outside-toplevel from onadata.apps.viewer.models.column_rename import ColumnRename self._variable_names = ColumnRename.get_dict() @@ -641,26 +664,29 @@ def get_list_of_parsed_instances(self, flat=True): # TODO: there is information we want to add in parsed xforms. yield i.get_dict(flat=flat) - def _rename_key(self, d, old_key, new_key): - if new_key in d: - raise AssertionError(d) - d[new_key] = d[old_key] - del d[old_key] - - def _expand_geocodes(self, d, key, e): - if e and e.bind.get("type") == "geopoint": - geodata = d[key].split() - for i in range(len(geodata)): - new_key = "%s_%s" % (key, self.geodata_suffixes[i]) - d[new_key] = geodata[i] + def _rename_key(self, item, old_key, new_key): + """Moves a value in item at old_key to new_key""" + if new_key in item: + raise AssertionError(item) + item[new_key] = item[old_key] + del item[old_key] + + def _expand_geocodes(self, item, key, elem): + """Expands a geopoint into latitude, longitude, altitude, precision.""" + if elem and elem.bind.get("type") == "geopoint": + geodata = item[key].split() + for i, v in enumerate(geodata): + new_key = f"{key}_{self.geodata_suffixes[i]}" + item[new_key] = v def get_data_for_excel(self): - for d in self.get_list_of_parsed_instances(): - for key in list(d): - e = self.get_element(key) - _expand_select_all_that_apply(d, key, e) - self._expand_geocodes(d, key, e) - yield d + """Returns submissions with select all and geopoint fields expanded""" + for row in self.get_list_of_parsed_instances(): + for key in list(row): + elem = self.get_element(key) + _expand_select_all_that_apply(row, key, elem) + self._expand_geocodes(row, key, elem) + yield row def _mark_start_time_boolean(self): starttime_substring = 'jr:preloadParams="start"' @@ -865,7 +891,7 @@ def _set_title(self): title_xml = self.title[:XFORM_TITLE_LENGTH] if isinstance(self.xml, b): self.xml = self.xml.decode("utf-8") - self.xml = title_pattern.sub("%s" % title_xml, self.xml) + self.xml = title_pattern.sub("{title_xml}", self.xml) self._set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( @@ -900,9 +926,12 @@ def _set_public_key_field(self): self._set_encrypted_field() def update(self, *args, **kwargs): - super(XForm, self).save(*args, **kwargs) + """Persists the form to the DB.""" + super().save(*args, **kwargs) - def save(self, *args, **kwargs): + # pylint: disable=too-many-branches + def save(self, *args, **kwargs): # noqa: MC0001 + """Sets additional form properties before saving to the DB""" update_fields = kwargs.get("update_fields") if update_fields: kwargs["update_fields"] = list(set(list(update_fields) + ["date_modified"])) @@ -969,10 +998,10 @@ def save(self, *args, **kwargs): ): raise XLSFormError( _( - "The XForm id_string provided exceeds %s characters." - ' Please change the "id_string" or "form_id" values' - "in settings sheet or reduce the file name if you do" - " not have a settings sheets." % self.MAX_ID_LENGTH + f"The XForm id_string provided exceeds {self.MAX_ID_LENGTH} characters." + f' Please change the "id_string" or "form_id" values' + f"in settings sheet or reduce the file name if you do" + f" not have a settings sheets." ) ) @@ -984,7 +1013,7 @@ def save(self, *args, **kwargs): self.description = conditional_escape(self.description) - super(XForm, self).save(*args, **kwargs) + super().save(*args, **kwargs) def __str__(self): return getattr(self, "id_string", "") @@ -1033,6 +1062,7 @@ def soft_delete(self, user=None): metadata.soft_delete() def submission_count(self, force_update=False): + """Returns the form's number of submission.""" if self.num_of_submissions == 0 or force_update: if self.is_merged_dataset: count = ( @@ -1049,7 +1079,7 @@ def submission_count(self, force_update=False): self.save(update_fields=["num_of_submissions"]) # clear cache - key = "{}{}".format(XFORM_COUNT, self.pk) + key = f"{XFORM_COUNT}{self.pk}" safe_delete(key) return self.num_of_submissions @@ -1079,6 +1109,7 @@ def geocoded_submission_count(self): ).count() def time_of_last_submission(self): + """Returns the timestamp of when the latest submission was created.""" if self.last_submission_time is None and self.num_of_submissions > 0: try: last_submission = self.instances.filter(deleted_at__isnull=True).latest( @@ -1092,25 +1123,35 @@ def time_of_last_submission(self): return self.last_submission_time def time_of_last_submission_update(self): + """Returns the timestamp of the last updated submission for the form.""" + last_submission_time = None try: # we also consider deleted instances in this case - return self.instances.latest("date_modified").date_modified + last_submission_time = self.instances.latest("date_modified").date_modified except ObjectDoesNotExist: pass + return last_submission_time + def get_hash(self): - return "md5:%s" % md5(self.xml.encode("utf8")).hexdigest() + """Returns the MD5 hash of the forms XML content prefixed by 'md5:'""" + return f"md5:{md5(self.xml.encode('utf8')).hexdigest()}" @property def can_be_replaced(self): + """Returns True if the form has zero submissions - forms with zero permissions + can be replaced.""" return self.num_of_submissions == 0 @classmethod def public_forms(cls): + """Returns a queryset of public forms i.e. shared = True""" return cls.objects.filter(shared=True) +# pylint: disable=unused-argument def update_profile_num_submissions(sender, instance, **kwargs): + """Reduce the user's number of submissions on deletions.""" profile_qs = User.profile.get_queryset() try: profile = profile_qs.select_for_update().get(pk=instance.user.profile.pk) @@ -1118,8 +1159,7 @@ def update_profile_num_submissions(sender, instance, **kwargs): pass else: profile.num_of_submissions -= instance.num_of_submissions - if profile.num_of_submissions < 0: - profile.num_of_submissions = 0 + profile.num_of_submissions = max(profile.num_of_submissions, 0) profile.save() @@ -1131,46 +1171,25 @@ def update_profile_num_submissions(sender, instance, **kwargs): def clear_project_cache(project_id): - safe_delete("{}{}".format(PROJ_OWNER_CACHE, project_id)) - safe_delete("{}{}".format(PROJ_FORMS_CACHE, project_id)) - safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, project_id)) - safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, project_id)) - safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, project_id)) - - -def set_object_permissions(sender, instance=None, created=False, **kwargs): - # clear cache - project = instance.project - project.refresh_from_db() - clear_project_cache(project.pk) - safe_delete("{}{}".format(IS_ORG, instance.pk)) - - if created: - from onadata.libs.permissions import OwnerRole - - OwnerRole.add(instance.user, instance) - - if instance.created_by and instance.user != instance.created_by: - OwnerRole.add(instance.created_by, instance) - - from onadata.libs.utils.project_utils import set_project_perms_to_xform - - set_project_perms_to_xform(instance, project) - - -post_save.connect( - set_object_permissions, sender=XForm, dispatch_uid="xform_object_permissions" -) + safe_delete(f"{PROJ_OWNER_CACHE}{project_id}") + safe_delete(f"{PROJ_FORMS_CACHE}{project_id}") + safe_delete(f"{PROJ_BASE_FORMS_CACHE}{project_id}") + safe_delete(f"{PROJ_SUB_DATE_CACHE}{project_id}") + safe_delete(f"{PROJ_NUM_DATASET_CACHE}{project_id}") +# pylint: disable=unused-argument def save_project(sender, instance=None, created=False, **kwargs): + """Update the date_modified field in the XForm's project.""" instance.project.save(update_fields=["date_modified"]) pre_save.connect(save_project, sender=XForm, dispatch_uid="save_project_xform") +# pylint: disable=unused-argument def xform_post_delete_callback(sender, instance, **kwargs): + """Clear project cache after deleting an XForm.""" if instance.project_id: clear_project_cache(instance.project_id) @@ -1200,7 +1219,7 @@ def check_xform_uuid(new_uuid): count = XForm.objects.filter(uuid=new_uuid, deleted_at__isnull=True).count() if count > 0: - raise DuplicateUUIDError("An xform with uuid: %s already exists" % new_uuid) + raise DuplicateUUIDError(f"An xform with uuid: {new_uuid} already exists") def update_xform_uuid(username, id_string, new_uuid): diff --git a/onadata/apps/logger/signals.py b/onadata/apps/logger/signals.py index 9e044efdab..316c0a998d 100644 --- a/onadata/apps/logger/signals.py +++ b/onadata/apps/logger/signals.py @@ -1,16 +1,27 @@ +# -*- coding: utf-8 -*- +""" +logger signals module +""" from django.db.models.signals import post_save from django.dispatch import receiver from onadata.apps.logger.models import MergedXForm +from onadata.apps.logger.models import XForm +from onadata.apps.logger.models.xform import clear_project_cache from onadata.libs.permissions import OwnerRole +from onadata.libs.utils.cache_tools import ( + IS_ORG, + safe_delete, +) from onadata.libs.utils.project_utils import set_project_perms_to_xform +# pylint: disable=unused-argument @receiver( - post_save, - sender=MergedXForm, - dispatch_uid='set_project_perms_to_merged_xform') -def set_object_permissions(sender, instance=None, created=False, **kwargs): + post_save, sender=MergedXForm, dispatch_uid="set_project_perms_to_merged_xform" +) +def set_project_object_permissions(sender, instance=None, created=False, **kwargs): + """Apply project permission to the merged form.""" if created: OwnerRole.add(instance.user, instance) OwnerRole.add(instance.user, instance.xform_ptr) @@ -21,3 +32,26 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): set_project_perms_to_xform(instance, instance.project) set_project_perms_to_xform(instance.xform_ptr, instance.project) + + +# pylint: disable=unused-argument +def set_xform_object_permissions(sender, instance=None, created=False, **kwargs): + """Apply project permissions to the user that created the form.""" + # clear cache + project = instance.project + project.refresh_from_db() + clear_project_cache(project.pk) + safe_delete(f"{IS_ORG}{instance.pk}") + + if created: + OwnerRole.add(instance.user, instance) + + if instance.created_by and instance.user != instance.created_by: + OwnerRole.add(instance.created_by, instance) + + set_project_perms_to_xform(instance, project) + + +post_save.connect( + set_xform_object_permissions, sender=XForm, dispatch_uid="xform_object_permissions" +) diff --git a/onadata/apps/viewer/models/column_rename.py b/onadata/apps/viewer/models/column_rename.py index 5e5efd17e0..7074460751 100644 --- a/onadata/apps/viewer/models/column_rename.py +++ b/onadata/apps/viewer/models/column_rename.py @@ -1,7 +1,15 @@ +# -*- coding: utf-8 -*- +""" +ColumnRename model +""" from django.db import models class ColumnRename(models.Model): + """ + ColumnRename model + """ + xpath = models.CharField(max_length=255, unique=True) column_name = models.CharField(max_length=32) @@ -10,4 +18,5 @@ class Meta: @classmethod def get_dict(cls): - return dict([(cr.xpath, cr.column_name) for cr in cls.objects.all()]) + """Returns a dictionary where xpath is key and column_name is value""" + return {cr.xpath: cr.column_name for cr in cls.objects.all()} From 1f4bc1959d0c3e2fa7136ecf8b29719cdb47a60d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 12:19:12 +0300 Subject: [PATCH 058/234] Cleanup --- .../tests/viewsets/test_abstract_viewset.py | 6 +- .../api/tests/viewsets/test_charts_viewset.py | 2 +- .../tests/viewsets/test_dataview_viewset.py | 1648 +++++++++-------- .../viewsets/test_merged_xform_viewset.py | 8 +- .../tests/viewsets/test_project_viewset.py | 223 ++- .../api/tests/viewsets/test_xform_viewset.py | 8 +- 6 files changed, 1031 insertions(+), 864 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index f5673a0f2a..9bee9ab54b 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -213,7 +213,7 @@ def _project_create(self, project_data={}, merge=True): self.project = Project.objects.filter(name=data["name"], created_by=self.user)[ 0 ] - data["url"] = "http://testserver/api/v1/projects/%s" % self.project.pk + data["url"] = f"http://testserver/api/v1/projects/{self.project.pk}" self.assertDictContainsSubset(data, response.data) request.user = self.user @@ -482,7 +482,7 @@ def _create_dataview(self, data=None, project=None, xform=None): ) self.assertEquals( response.data["url"], - "http://testserver/api/v1/dataviews/%s" % self.data_view.pk, + f"http://testserver/api/v1/dataviews/{self.data_view.pk}", ) def _create_widget(self, data=None, group_by=""): @@ -491,7 +491,7 @@ def _create_widget(self, data=None, group_by=""): if not data: data = { "title": "Widget that", - "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "content_object": f"http://testserver/api/v1/forms/{self.xform.pk}", "description": "Test widget", "aggregation": "Sum", "widget_type": "charts", diff --git a/onadata/apps/api/tests/viewsets/test_charts_viewset.py b/onadata/apps/api/tests/viewsets/test_charts_viewset.py index 6a50d78273..e340e5c446 100644 --- a/onadata/apps/api/tests/viewsets/test_charts_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_charts_viewset.py @@ -94,7 +94,7 @@ def test_correct_merged_dataset_data_for_charts(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } # anonymous user request = self.factory.post('/', data=data) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index dd587ed5c9..8b8b7fd7ee 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -1,7 +1,8 @@ +# -*- coding: utf-8 -*- +"""Test DataViewViewSet""" import json import os import csv -from builtins import open from datetime import datetime, timedelta from django.conf import settings @@ -14,8 +15,7 @@ from onadata.libs.permissions import ReadOnlyRole from onadata.apps.logger.models.data_view import DataView -from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.viewer.models.export import Export from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.dataview_viewset import DataViewViewSet @@ -24,83 +24,100 @@ from onadata.libs.utils.cache_tools import ( DATAVIEW_COUNT, DATAVIEW_LAST_SUBMISSION_TIME, - PROJECT_LINKED_DATAVIEWS) + PROJECT_LINKED_DATAVIEWS, +) from onadata.libs.utils.common_tags import EDITED, MONGO_STRFTIME from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.libs.utils.common_tools import ( - filename_from_disposition, get_response_content) + filename_from_disposition, + get_response_content, +) class TestDataViewViewSet(TestAbstractViewSet): - def setUp(self): - super(TestDataViewViewSet, self).setUp() + super().setUp() xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) - for x in range(1, 9): + for fixture in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + f"uuid{fixture}", + "submission.xml", + ) self._make_submission(path) - x += 1 - self.view = DataViewViewSet.as_view({ - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update', - 'delete': 'destroy', - 'get': 'retrieve' - }) + self.view = DataViewViewSet.as_view( + { + "post": "create", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + "get": "retrieve", + } + ) def test_create_dataview(self): self._create_dataview() + # pylint: disable=invalid-name def test_dataview_with_attachment_field(self): - view = DataViewViewSet.as_view({ - 'get': 'data' - }) + view = DataViewViewSet.as_view({"get": "data"}) media_file = "test-image.png" attachment_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - media_file) + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", media_file + ) submission_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid10', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid10", + "submission.xml", + ) # make a submission with an attachment - with open(attachment_file_path, 'rb') as f: + with open(attachment_file_path, "rb") as f: self._make_submission(submission_file_path, media_file=f) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", # ensure there's an attachment column(photo) in you dataview - 'columns': '["name", "age", "gender", "photo"]' + "columns": '["name", "age", "gender", "photo"]', } self._create_dataview(data=data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - for a in response.data: + for item in response.data: # retrieve the instance with attachment - if a.get('photo') == media_file: - instance_with_attachment = a + if item.get("photo") == media_file: + instance_with_attachment = item self.assertTrue(instance_with_attachment) - attachment_info = instance_with_attachment.get('_attachments')[0] - self.assertEquals(u'image/png', attachment_info.get(u'mimetype')) - self.assertEquals( - u'{}/attachments/{}_{}/{}'.format( - self.user.username, self.xform.id, self.xform.id_string, - media_file), - attachment_info.get(u'filename')) - self.assertEquals(response.status_code, 200) + attachment_info = instance_with_attachment.get("_attachments")[0] + self.assertEqual("image/png", attachment_info.get("mimetype")) + self.assertEqual( + f"{self.user.username}/attachments/{self.xform.id}_{self.xform.id_string}/{media_file}", + attachment_info.get("filename"), + ) + self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_get_dataview_form_definition(self): self._create_dataview() @@ -111,29 +128,33 @@ def test_get_dataview_form_definition(self): "id_string": "tutorial", "type": "survey", } - self.view = DataViewViewSet.as_view({ - 'get': 'form', - }) - request = self.factory.get('/', **self.extra) + self.view = DataViewViewSet.as_view( + { + "get": "form", + } + ) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # JSON format - response = self.view(request, pk=self.data_view.pk, format='json') + response = self.view(request, pk=self.data_view.pk, format="json") self.assertEqual(response.status_code, 200) self.assertDictContainsSubset(data, response.data) def test_get_dataview_form_details(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'form_details', - }) - request = self.factory.get('/', **self.extra) + self.view = DataViewViewSet.as_view( + { + "get": "form_details", + } + ) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - response = self.view(request, pk=self.data_view.pk, format='json') + response = self.view(request, pk=self.data_view.pk, format="json") self.assertEqual(response.status_code, 200) self.assertIn("title", response.data) @@ -144,80 +165,84 @@ def test_get_dataview_form_details(self): def test_get_dataview(self): self._create_dataview() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView') - self.assertEquals(response.data['xform'], - 'http://testserver/api/v1/forms/%s' % self.xform.pk) - self.assertEquals(response.data['project'], - 'http://testserver/api/v1/projects/%s' - % self.project.pk) - self.assertEquals(response.data['columns'], - ["name", "age", "gender"]) - self.assertEquals(response.data['query'], - [{"column": "age", "filter": ">", "value": "20"}, - {"column": "age", "filter": "<", "value": "50"}]) - self.assertEquals(response.data['url'], - 'http://testserver/api/v1/dataviews/%s' - % self.data_view.pk) - self.assertEquals(response.data['last_submission_time'], - '2015-03-09T13:34:05') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView") + self.assertEqual( + response.data["xform"], f"http://testserver/api/v1/forms/{self.xform.pk}" + ) + self.assertEqual( + response.data["project"], + f"http://testserver/api/v1/projects/{self.project.pk}", + ) + self.assertEqual(response.data["columns"], ["name", "age", "gender"]) + self.assertEqual( + response.data["query"], + [ + {"column": "age", "filter": ">", "value": "20"}, + {"column": "age", "filter": "<", "value": "50"}, + ], + ) + self.assertEqual( + response.data["url"], + f"http://testserver/api/v1/dataviews/{self.data_view.pk}", + ) + self.assertEqual(response.data["last_submission_time"], "2015-03-09T13:34:05") # Public self.project.shared = True self.project.save() - anon_request = self.factory.get('/') + anon_request = self.factory.get("/") anon_response = self.view(anon_request, pk=self.data_view.pk) - self.assertEquals(anon_response.status_code, 200) + self.assertEqual(anon_response.status_code, 200) # Private self.project.shared = False self.project.save() - anon_request = self.factory.get('/') + anon_request = self.factory.get("/") anon_response = self.view(anon_request, pk=self.data_view.pk) - self.assertEquals(anon_response.status_code, 404) + self.assertEqual(anon_response.status_code, 404) def test_update_dataview(self): self._create_dataview() data = { - 'name': "My DataView updated", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView updated", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], 'My DataView updated') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["name"], "My DataView updated") - self.assertEquals(response.data['columns'], - ["name", "age", "gender"]) + self.assertEqual(response.data["columns"], ["name", "age", "gender"]) - self.assertEquals(response.data['query'], - [{"column": "age", "filter": ">", "value": "20"}]) + self.assertEqual( + response.data["query"], [{"column": "age", "filter": ">", "value": "20"}] + ) def test_patch_dataview(self): self._create_dataview() data = { - 'name': "My DataView updated", + "name": "My DataView updated", } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], 'My DataView updated') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["name"], "My DataView updated") def test_soft_delete_dataview(self): """ @@ -228,13 +253,13 @@ def test_soft_delete_dataview(self): self.assertIsNone(self.data_view.deleted_at) self.assertNotIn("-deleted-at-", self.data_view.name) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=dataview_id) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['name'], u'My DataView') + self.assertEqual(response.data["name"], "My DataView") # delete - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=dataview_id) self.assertEqual(response.status_code, 204) @@ -242,118 +267,121 @@ def test_soft_delete_dataview(self): data_view = DataView.objects.get(pk=dataview_id) self.assertIsNotNone(data_view.deleted_at) self.assertIn("-deleted-at-", data_view.name) - self.assertEqual(data_view.deleted_by.username, u'bob') + self.assertEqual(data_view.deleted_by.username, "bob") + # pylint: disable=invalid-name def test_soft_deleted_dataview_not_in_forms_list(self): self._create_dataview() - get_form_request = self.factory.get('/', **self.extra) + get_form_request = self.factory.get("/", **self.extra) xform_serializer = XFormSerializer( - self.xform, - context={'request': get_form_request}) + self.xform, context={"request": get_form_request} + ) - self.assertIsNotNone(xform_serializer.data['data_views']) + self.assertIsNotNone(xform_serializer.data["data_views"]) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) xform_serializer = XFormSerializer( - self.xform, - context={'request': get_form_request}) - self.assertEquals(xform_serializer.data['data_views'], []) + self.xform, context={"request": get_form_request} + ) + self.assertEqual(xform_serializer.data["data_views"], []) + # pylint: disable=invalid-name def test_soft_deleted_dataview_not_in_project(self): """ Test that once a filtered dataset is soft deleted it does not appear in the list of forms for a project """ self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) + view = ProjectViewSet.as_view({"get": "retrieve"}) # assert that dataview is in the returned list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertIsNotNone(response.data['data_views']) + self.assertIsNotNone(response.data["data_views"]) # delete dataview - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) # assert that deleted dataview is not in the returned list - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEqual(response.data['data_views'], []) + self.assertEqual(response.data["data_views"], []) def test_list_dataview(self): self._create_dataview() data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'list', - }) + view = DataViewViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) - anon_request = request = self.factory.get('/') + anon_request = request = self.factory.get("/") anon_response = view(anon_request) - self.assertEquals(anon_response.status_code, 401) + self.assertEqual(anon_response.status_code, 401) def test_get_dataview_no_perms(self): self._create_dataview() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 404) + self.assertEqual(response.status_code, 404) # assign alice the perms ReadOnlyRole.add(self.user, self.data_view.project) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_dataview_data_filter_integer(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 3) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 3) self.assertIn("_id", response.data[0]) def test_dataview_data_filter_decimal(self): @@ -362,201 +390,227 @@ def test_dataview_data_filter_decimal(self): """ # publish form with decimal field and make submissions path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "age_decimal", "age_decimal.xlsx") + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "age_decimal", + "age_decimal.xlsx", + ) self._publish_xls_form_to_project(xlsform_path=path) - for x in range(1, 3): + for fixture in range(1, 3): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'age_decimal', 'instances', 'submission{}.xml'.format(x), ) + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "age_decimal", + "instances", + f"submission{fixture}.xml", + ) self._make_submission(path) - x += 1 # create a dataview using filter age > 30 data = { - 'name': "My Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age"]', - 'query': '[{"column":"age","filter":">","value":"30"}]' + "name": "My Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age"]', + "query": '[{"column":"age","filter":">","value":"30"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) - request = self.factory.get('/', **self.extra) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['age'], 31) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data[0]["age"], 31) def test_dataview_data_filter_date(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': '[{"column":"_submission_time",' - '"filter":">=","value":"2015-01-01T00:00:00"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": '[{"column":"_submission_time",' + '"filter":">=","value":"2015-01-01T00:00:00"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 7) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 7) self.assertIn("_id", response.data[0]) + # pylint: disable=invalid-name def test_dataview_data_filter_string(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': '[{"column":"gender","filter":"<>","value":"male"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": '[{"column":"gender","filter":"<>","value":"male"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + # pylint: disable=invalid-name def test_dataview_data_filter_condition(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"name","filter":"=","value":"Fred",' - ' "condition":"or"},' - '{"column":"name","filter":"=","value":"Kameli",' - ' "condition":"or"},' - '{"column":"gender","filter":"=","value":"male"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"name","filter":"=","value":"Fred",' + ' "condition":"or"},' + '{"column":"name","filter":"=","value":"Kameli",' + ' "condition":"or"},' + '{"column":"gender","filter":"=","value":"male"}]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) self.assertIn("_id", response.data[0]) def test_dataview_invalid_filter(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"name","filter":"<=>","value":"Fred",' - ' "condition":"or"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"name","filter":"<=>","value":"Fred",' + ' "condition":"or"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - {'query': [u'Filter not supported']}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, {"query": ["Filter not supported"]}) def test_dataview_sql_injection(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "age"]', - 'query': '[{"column":"age","filter":"=",' - '"value":"1;UNION ALL SELECT NULL,version()' - ',NULL LIMIT 1 OFFSET 1--;"}]' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "age"]', + "query": '[{"column":"age","filter":"=",' + '"value":"1;UNION ALL SELECT NULL,version()' + ',NULL LIMIT 1 OFFSET 1--;"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertIn('detail', response.data) + self.assertEqual(response.status_code, 400) + self.assertIn("detail", response.data) - self.assertTrue(str(response.data.get('detail')) - .startswith("invalid input syntax for integer")) + self.assertTrue( + str(response.data.get("detail")).startswith( + "invalid input syntax for integer" + ) + ) def test_dataview_invalid_columns(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': 'age' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": "age", } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertIn(response.data['columns'][0], - ['Expecting value: line 1 column 1 (char 0)', - 'No JSON object could be decoded']) + self.assertEqual(response.status_code, 400) + self.assertIn( + response.data["columns"][0], + [ + "Expecting value: line 1 column 1 (char 0)", + "No JSON object could be decoded", + ], + ) def test_dataview_invalid_query(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', - 'query': 'age=10' + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', + "query": "age=10", } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertIn(response.data['query'][0], - ['Expecting value: line 1 column 1 (char 0)', - u'No JSON object could be decoded']) + self.assertEqual(response.status_code, 400) + self.assertIn( + response.data["query"][0], + [ + "Expecting value: line 1 column 1 (char 0)", + "No JSON object could be decoded", + ], + ) + # pylint: disable=invalid-name def test_dataview_query_not_required(self): data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 8) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 8) self.assertIn("_id", response.data[0]) self.assertIn(EDITED, response.data[0]) @@ -565,209 +619,226 @@ def test_csv_export_dataview(self): self._create_dataview() count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) - test_file_path = os.path.join(settings.PROJECT_ROOT, 'apps', - 'viewer', 'tests', 'fixtures', - 'dataview.csv') - with open(test_file_path, encoding='utf-8') as test_file: + test_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "viewer", "tests", "fixtures", "dataview.csv" + ) + with open(test_file_path, encoding="utf-8") as test_file: self.assertEqual(content, test_file.read()) def test_csvzip_export_dataview(self): self._create_dataview() count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csvzip') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csvzip") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) - request = self.factory.get('/', **self.extra) - response = view(request, pk='[invalid pk]', format='csvzip') + request = self.factory.get("/", **self.extra) + response = view(request, pk="[invalid pk]", format="csvzip") self.assertEqual(response.status_code, 404) def test_zip_export_dataview(self): media_file = "test-image.png" attachment_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - media_file) + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", media_file + ) submission_file_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid10', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid10", + "submission.xml", + ) # make a submission with an attachment - with open(attachment_file_path, 'rb') as f: + with open(attachment_file_path, "rb") as f: self._make_submission(submission_file_path, media_file=f) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "photo"]', - 'query': '[{"column":"age","filter":"=","value":"90"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "photo"]', + "query": '[{"column":"age","filter":"=","value":"90"}]', } self._create_dataview(data) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='zip') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="zip") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".zip") + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_csv_dataview_data_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv"}, - **self.extra) + request = self.factory.get("/", data={"format": "csv"}, **self.extra) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_csv_dataview_with_labels_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv", - 'include_labels': 'true'}, - **self.extra) + request = self.factory.get( + "/", data={"format": "csv", "include_labels": "true"}, **self.extra + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) - with default_storage.open(export.filepath, 'r') as f: + with default_storage.open(export.filepath, "r") as f: csv_reader = csv.reader(f) next(csv_reader) labels = next(csv_reader) - self.assertIn( - 'Gender', labels - ) + self.assertIn("Gender", labels) + # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_xls_dataview_with_labels_async(self, async_result): self._create_dataview() self._publish_xls_form_to_project() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "xls", - "force_xlsx": 'true', - 'include_labels': 'true'}, - **self.extra) + request = self.factory.get( + "/", + data={"format": "xls", "force_xlsx": "true", "include_labels": "true"}, + **self.extra, + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) @@ -776,201 +847,206 @@ def test_export_xls_dataview_with_labels_async(self, async_result): workbook = load_workbook(export.full_filepath) sheet_name = workbook.get_sheet_names()[0] main_sheet = workbook.get_sheet_by_name(sheet_name) - self.assertIn('Gender', tuple(main_sheet.values)[1]) + self.assertIn("Gender", tuple(main_sheet.values)[1]) self.assertEqual(len(tuple(main_sheet.values)), 5) - def _test_csv_export_with_hxl_support(self, name, columns, expected_output): # noqa + def _test_csv_export_with_hxl_support(self, name, columns, expected_output): # noqa data = { - 'name': name, - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': columns, - 'query': '[]' + "name": name, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": columns, + "query": "[]", } self._create_dataview(data=data) dataview_pk = DataView.objects.last().pk - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) request = self.factory.get( - '/', data={"format": "csv", "include_hxl": True}, **self.extra) + "/", data={"format": "csv", "include_hxl": True}, **self.extra + ) response = view(request, pk=dataview_pk) - self.assertIsNotNone( - next(response.streaming_content), - expected_output - ) + self.assertIsNotNone(next(response.streaming_content), expected_output) + # pylint: disable=invalid-name def test_csv_export_with_hxl_support(self): self._publish_form_with_hxl_support() self._test_csv_export_with_hxl_support( - 'test name 1', - '["name"]', - 'name\nCristiano Ronaldo 1\nLionel Messi\n' + "test name 1", '["name"]', "name\nCristiano Ronaldo 1\nLionel Messi\n" ) self._test_csv_export_with_hxl_support( - 'test name 2', - '["age"]', - 'age\n#age,\n31\n29\n' + "test name 2", '["age"]', "age\n#age,\n31\n29\n" ) self._test_csv_export_with_hxl_support( - 'test name 3', + "test name 3", '["age", "name"]', - 'age,name\n#age,\n31,Cristiano Ronaldo\n29,Lionel Messi\n' + "age,name\n#age,\n31,Cristiano Ronaldo\n29,Lionel Messi\n", ) def test_get_charts_data(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) data_view_data = DataView.query_data(self.data_view) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) - data = {'field_xpath': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_xpath": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) + # pylint: disable=invalid-name def test_get_charts_data_for_submission_time_field(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - data = {'field_name': '_submission_time'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_submission_time"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'datetime') - self.assertEqual(response.data['field_name'], '_submission_time') - self.assertEqual(response.data['data_type'], 'time_based') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "datetime") + self.assertEqual(response.data["field_name"], "_submission_time") + self.assertEqual(response.data["data_type"], "time_based") - data = {'field_name': '_submitted_by'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_submitted_by"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'text') - self.assertEqual(response.data['field_name'], '_submitted_by') - self.assertEqual(response.data['data_type'], 'categorized') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "text") + self.assertEqual(response.data["field_name"], "_submitted_by") + self.assertEqual(response.data["data_type"], "categorized") - data = {'field_name': '_duration'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "_duration"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], '_duration') - self.assertEqual(response.data['data_type'], 'numeric') + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "_duration") + self.assertEqual(response.data["data_type"], "numeric") def test_get_charts_data_for_grouped_field(self): data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "a_group/grouped"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "a_group/grouped"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'grouped'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "grouped"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'select one') - self.assertEqual(response.data['field_name'], 'grouped') - self.assertEqual(response.data['data_type'], 'categorized') - self.assertEqual(len(response.data['data']), 2) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "select one") + self.assertEqual(response.data["field_name"], "grouped") + self.assertEqual(response.data["data_type"], "categorized") + self.assertEqual(len(response.data["data"]), 2) + # pylint: disable=invalid-name def test_get_charts_data_field_not_in_dataview_columns(self): self._create_dataview() - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) - data = {'field_name': 'grouped'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "grouped"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 404) def test_get_charts_data_with_empty_query(self): data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": "[]", } self._create_dataview(data) - self.view = DataViewViewSet.as_view({ - 'get': 'charts', - }) + self.view = DataViewViewSet.as_view( + { + "get": "charts", + } + ) data_view_data = DataView.query_data(self.data_view) - request = self.factory.get('/charts', **self.extra) + request = self.factory.get("/charts", **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - data = {'field_name': 'age'} - request = self.factory.get('/charts', data, **self.extra) + data = {"field_name": "age"} + request = self.factory.get("/charts", data, **self.extra) response = self.view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data['field_type'], 'integer') - self.assertEqual(response.data['field_name'], 'age') - self.assertEqual(response.data['data_type'], 'numeric') - self.assertEqual(len(response.data['data']), len(data_view_data)) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data["field_type"], "integer") + self.assertEqual(response.data["field_name"], "age") + self.assertEqual(response.data["data_type"], "numeric") + self.assertEqual(len(response.data["data"]), len(data_view_data)) def test_geopoint_dataview(self): # Dataview with geolocation column selected. # -> instances_with_geopoints= True data = { - 'name': "My DataView1", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "location"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView1", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "location"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) @@ -979,88 +1055,101 @@ def test_geopoint_dataview(self): # Dataview with geolocation column NOT selected # -> instances_with_geopoints= False data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) self.assertFalse(self.data_view.instances_with_geopoints) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView2') - self.assertEquals(response.data['instances_with_geopoints'], False) - - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView2") + self.assertEqual(response.data["instances_with_geopoints"], False) + + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertNotIn("location", response.data[0]) self.assertNotIn("_geolocation", response.data[0]) + # pylint: disable=invalid-name def test_geopoint_submission_dataview(self): data = { - 'name': "My DataView3", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender", "location"]', - 'query': '[{"column":"age","filter":">=","value":"87"}]' + "name": "My DataView3", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender", "location"]', + "query": '[{"column":"age","filter":">=","value":"87"}]', } self._create_dataview(data) self.assertTrue(self.data_view.instances_with_geopoints) # make submission with geopoint - path = os.path.join(settings.PROJECT_ROOT, 'libs', 'tests', "utils", - 'fixtures', 'tutorial', 'instances', - 'uuid{}'.format(9), 'submission.xml') + path = os.path.join( + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid9", + "submission.xml", + ) self._make_submission(path) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['dataviewid'], self.data_view.pk) - self.assertEquals(response.data['name'], 'My DataView3') - self.assertEquals(response.data['instances_with_geopoints'], True) - - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["dataviewid"], self.data_view.pk) + self.assertEqual(response.data["name"], "My DataView3") + self.assertEqual(response.data["instances_with_geopoints"], True) + + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 200) self.assertIn("location", response.data[0]) self.assertIn("_geolocation", response.data[0]) + # pylint: disable=invalid-name def test_dataview_project_cache_cleared(self): self._create_dataview() - view = ProjectViewSet.as_view({ - 'get': 'retrieve', - }) + view = ProjectViewSet.as_view( + { + "get": "retrieve", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - cached_dataviews = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + cached_dataviews = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNotNone(cached_dataviews) @@ -1068,344 +1157,376 @@ def test_dataview_project_cache_cleared(self): self.data_view.name = "updated name" self.data_view.save() - updated_cache = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + updated_cache = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNone(updated_cache) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - cached_dataviews = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + cached_dataviews = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNotNone(cached_dataviews) self.data_view.delete() - updated_cache = cache.get('{}{}'.format(PROJECT_LINKED_DATAVIEWS, - self.project.pk)) + updated_cache = cache.get(f"{PROJECT_LINKED_DATAVIEWS}{self.project.pk}") self.assertIsNone(updated_cache) + # pylint: disable=invalid-name def test_dataview_update_refreshes_cached_data(self): self._create_dataview() - cache.set('{}{}'.format(DATAVIEW_COUNT, self.data_view.xform.pk), 5) - cache.set('{}{}'.format(DATAVIEW_LAST_SUBMISSION_TIME, - self.data_view.xform.pk), - '2015-03-09T13:34:05') + cache.set(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}", 5) + cache.set( + f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}", + "2015-03-09T13:34:05", + ) self.data_view.name = "Updated Dataview" self.data_view.save() + self.assertIsNone(cache.get(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}")) self.assertIsNone( - cache.get('{}{}'.format(DATAVIEW_COUNT, self.data_view.xform.pk))) - self.assertIsNone(cache.get('{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, self.data_view.xform.pk))) + cache.get(f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}") + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) expected_count = 3 expected_last_submission_time = "2015-03-09T13:34:05" - self.assertEquals(response.data['count'], expected_count) - self.assertEquals(response.data['last_submission_time'], - '2015-03-09T13:34:05') + self.assertEqual(response.data["count"], expected_count) + self.assertEqual(response.data["last_submission_time"], "2015-03-09T13:34:05") - cache_dict = cache.get('{}{}'.format(DATAVIEW_COUNT, - self.data_view.xform.pk)) - self.assertEquals(cache_dict.get(self.data_view.pk), expected_count) - self.assertEquals(cache.get('{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, self.data_view.xform.pk)), - expected_last_submission_time) + cache_dict = cache.get(f"{DATAVIEW_COUNT}{self.data_view.xform.pk}") + self.assertEqual(cache_dict.get(self.data_view.pk), expected_count) + self.assertEqual( + cache.get(f"{DATAVIEW_LAST_SUBMISSION_TIME}{self.data_view.xform.pk}"), + expected_last_submission_time, + ) + # pylint: disable=invalid-name def test_export_dataview_not_affected_by_normal_exports(self): count = Export.objects.all().count() - view = XFormViewSet.as_view({ - 'get': 'retrieve', - }) + view = XFormViewSet.as_view( + { + "get": "retrieve", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 2, Export.objects.all().count()) + self.assertEqual(count + 2, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) # count csv headers and ensure they are three - self.assertEqual(len(content.split('\n')[0].split(',')), 3) + self.assertEqual(len(content.split("\n")[0].split(",")), 3) def test_matches_parent(self): self._create_dataview() self.assertFalse(self.data_view.matches_parent) - columns = [u'name', u'age', u'gender', u'photo', u'date', u'location', - u'pizza_fan', u'pizza_hater', u'pizza_type', - u'favorite_toppings', u'test_location2.latitude', - u'test_location2.longitude', u'test_location.precision', - u'test_location2.precision', u'test_location.altitude', - u'test_location.latitude', u'test_location2.altitude', - u'test_location.longitude', u'thanks', u'a_group', - u'a_group/grouped', u'a_group/a_text', u'start_time', - u'end_time', u'today', u'imei', u'phonenumber', - 'meta', 'meta/instanceID'] + columns = [ + "name", + "age", + "gender", + "photo", + "date", + "location", + "pizza_fan", + "pizza_hater", + "pizza_type", + "favorite_toppings", + "test_location2.latitude", + "test_location2.longitude", + "test_location.precision", + "test_location2.precision", + "test_location.altitude", + "test_location.latitude", + "test_location2.altitude", + "test_location.longitude", + "thanks", + "a_group", + "a_group/grouped", + "a_group/a_text", + "start_time", + "end_time", + "today", + "imei", + "phonenumber", + "meta", + "meta/instanceID", + ] data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': json.dumps(columns), - 'query': '[{"column":"age","filter":">","value":"20"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": json.dumps(columns), + "query": '[{"column":"age","filter":">","value":"20"}]', } self._create_dataview(data) self.assertTrue(self.data_view.matches_parent) + # pylint: disable=invalid-name def test_dataview_create_data_filter_invalid_date(self): - invalid_query = '[{"column":"_submission_time",' \ - '"filter":">","value":"30/06/2015"}]' + invalid_query = ( + '[{"column":"_submission_time","filter":">","value":"30/06/2015"}]' + ) data = { - 'name': "Transportation Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "gender", "_submission_time"]', - 'query': invalid_query + "name": "Transportation Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "gender", "_submission_time"]', + "query": invalid_query, } - view = DataViewViewSet.as_view({ - 'get': 'data', - 'post': 'create', - 'patch': 'partial_update' - }) + view = DataViewViewSet.as_view( + {"get": "data", "post": "create", "patch": "partial_update"} + ) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) # Confirm you cannot create an invalid dataview - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_update_data_filter_invalid_date(self): - invalid_query = '[{"column":"_submission_time",' \ - '"filter":">","value":"30/06/2015"}]' + invalid_query = ( + '[{"column":"_submission_time","filter":">","value":"30/06/2015"}]' + ) self._create_dataview() - data = {'query': invalid_query} - request = self.factory.patch('/', data=data, **self.extra) + data = {"query": invalid_query} + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.data_view.pk) # Confirm you cannot update an invalid dataview - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_serializer_exception(self): invalid_query = [ - {"column": "_submission_time", - "filter": ">", - "value": "30/06/2015"} + {"column": "_submission_time", "filter": ">", "value": "30/06/2015"} ] self._create_dataview() self.data_view.query = invalid_query self.data_view.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) + # pylint: disable=invalid-name def test_dataview_notes_added_to_data(self): # Create note - view = NoteViewSet.as_view({ - 'post': 'create' - }) - comment = u"Dataview note" - note = {'note': comment} - data_id = self.xform.instances.all().order_by('pk')[0].pk - note['instance'] = data_id - request = self.factory.post('/', data=note, **self.extra) + view = NoteViewSet.as_view({"post": "create"}) + comment = "Dataview note" + note = {"note": comment} + data_id = self.xform.instances.all().order_by("pk")[0].pk + note["instance"] = data_id + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = view(request) self.assertEqual(response.status_code, 201) # Get dataview with added notes data = { - 'name': "My Dataview", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["age"]', + "name": "My Dataview", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["age"]', } self._create_dataview(data=data) - view = DataViewViewSet.as_view({ - 'get': 'data' - }) - request = self.factory.get('/', **self.extra) + view = DataViewViewSet.as_view({"get": "data"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 8) - data_with_notes = next(( - d for d in response.data if d["_id"] == data_id)) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 8) + data_with_notes = next((d for d in response.data if d["_id"] == data_id)) self.assertIn("_notes", data_with_notes) - self.assertEquals([{'created_by': self.user.id, - 'id': 1, - 'instance_field': None, - 'note': comment, - 'owner': self.user.username}], - data_with_notes["_notes"]) + self.assertEqual( + [ + { + "created_by": self.user.id, + "id": 1, + "instance_field": None, + "note": comment, + "owner": self.user.username, + } + ], + data_with_notes["_notes"], + ) def test_sort_dataview_data(self): self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) data = {"sort": '{"age": -1}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) - self.assertTrue( - self.is_sorted_desc([r.get("age") for r in response.data])) + self.assertEqual(response.status_code, 200) + self.assertTrue(self.is_sorted_desc([r.get("age") for r in response.data])) def test_invalid_date_filter(self): - view = DataViewViewSet.as_view({ - 'get': 'retrieve', - 'post': 'create', - }) + view = DataViewViewSet.as_view( + { + "get": "retrieve", + "post": "create", + } + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"26-01-2016"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"26-01-2016"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Date value in _submission_time should be' - u' yyyy-mm-ddThh:m:s or yyyy-mm-dd'] - }) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + { + "non_field_errors": [ + "Date value in _submission_time should be" + " yyyy-mm-ddThh:m:s or yyyy-mm-dd" + ] + }, + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"26/01/2016"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"26/01/2016"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, - { - u'non_field_errors': - [u'Date value in _submission_time should be' - u' yyyy-mm-ddThh:m:s or yyyy-mm-dd'] - }) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + { + "non_field_errors": [ + "Date value in _submission_time should be" + " yyyy-mm-ddThh:m:s or yyyy-mm-dd" + ] + }, + ) data = { - 'name': "My DataView", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"2016-01-16T00:00:00"}]' + "name": "My DataView", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"2016-01-16T00:00:00"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) data = { - 'name': "My DataView2", - 'xform': 'http://testserver/api/v1/forms/%s' % self.xform.pk, - 'project': 'http://testserver/api/v1/projects/%s' - % self.project.pk, - 'columns': '["name", "age", "gender"]', - 'query': '[{"column":"_submission_time","filter":">",' - '"value":"2016-01-16"}]' + "name": "My DataView2", + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", + "columns": '["name", "age", "gender"]', + "query": '[{"column":"_submission_time","filter":">",' + '"value":"2016-01-16"}]', } - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_search_dataview_data(self): self._create_dataview() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) data = {"query": "Fred"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual("Fred", response.data[0].get('name')) + self.assertEqual("Fred", response.data[0].get("name")) data = {"query": '{"age": 22}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual(22, response.data[0].get('age')) + self.assertEqual(22, response.data[0].get("age")) data = {"query": '{"age": {"$gte": 30}}'} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(1, len(response.data)) - self.assertEqual(45, response.data[0].get('age')) + self.assertEqual(45, response.data[0].get("age")) def test_invalid_url_parameters(self): - response = self.client.get('/api/v1/dataviews/css/ona.css/') - self.assertEquals(response.status_code, 404) + response = self.client.get("/api/v1/dataviews/css/ona.css/") + self.assertEqual(response.status_code, 404) + # pylint: disable=invalid-name,too-many-locals @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_export_xls_dataview_with_date_filter(self, async_result): """ Test dataview export with a date filter. @@ -1415,38 +1536,48 @@ def test_export_xls_dataview_with_date_filter(self, async_result): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' - - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) - - request = self.factory.get('/', data={"format": "xls", - "force_xlsx": 'true', - 'include_labels': 'true', - 'query': query_str}, - **self.extra) + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) + + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) + + request = self.factory.get( + "/", + data={ + "format": "xls", + "force_xlsx": "true", + "include_labels": "true", + "query": query_str, + }, + **self.extra, + ) response = view(request, pk=self.data_view.pk) self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) @@ -1455,7 +1586,7 @@ def test_export_xls_dataview_with_date_filter(self, async_result): workbook = load_workbook(export.full_filepath) sheet_name = workbook.get_sheet_names()[0] main_sheet = workbook.get_sheet_by_name(sheet_name) - self.assertIn('Gender', tuple(main_sheet.values)[1]) + self.assertIn("Gender", tuple(main_sheet.values)[1]) self.assertEqual(len(tuple(main_sheet.values)), 3) def test_csv_export_dataview_date_filter(self): @@ -1467,34 +1598,40 @@ def test_csv_export_dataview_date_filter(self): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'data', - }) + view = DataViewViewSet.as_view( + { + "get": "data", + } + ) - request = self.factory.get('/', data={'query': query_str}, - **self.extra) - response = view(request, pk=self.data_view.pk, format='csv') + request = self.factory.get("/", data={"query": query_str}, **self.extra) + response = view(request, pk=self.data_view.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) - basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + _basename, ext = os.path.splitext(filename) + self.assertEqual(ext, ".csv") content = get_response_content(response) - self.assertEqual(content, 'name,age,gender\nDennis Wambua,28,male\n') + self.assertEqual(content, "name,age,gender\nDennis Wambua,28,male\n") + # pylint: disable=too-many-locals @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.api.viewsets.dataview_viewset.AsyncResult') + @patch("onadata.apps.api.viewsets.dataview_viewset.AsyncResult") def test_csv_export_async_dataview_date_filter(self, async_result): """ Test dataview csv export async with a date filter. @@ -1504,43 +1641,48 @@ def test_csv_export_async_dataview_date_filter(self, async_result): start_date = datetime(2014, 9, 12, tzinfo=utc) first_datetime = start_date.strftime(MONGO_STRFTIME) second_datetime = start_date + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) count = Export.objects.all().count() - view = DataViewViewSet.as_view({ - 'get': 'export_async', - }) + view = DataViewViewSet.as_view( + { + "get": "export_async", + } + ) - request = self.factory.get('/', data={"format": "csv", - 'query': query_str}, - **self.extra) + request = self.factory.get( + "/", data={"format": "csv", "query": query_str}, **self.extra + ) response = view(request, pk=self.data_view.pk) self.assertEqual(response.status_code, 202) self.assertIsNotNone(response.data) - self.assertTrue('job_uuid' in response.data) - task_id = response.data.get('job_uuid') - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertTrue("job_uuid" in response.data) + task_id = response.data.get("job_uuid") + self.assertEqual(count + 1, Export.objects.all().count()) - export_pk = Export.objects.all().order_by('pk').reverse()[0].pk + export_pk = Export.objects.all().order_by("pk").reverse()[0].pk # metaclass for mocking results - job = type('AsyncResultMock', (), - {'state': 'SUCCESS', 'result': export_pk}) + job = type("AsyncResultMock", (), {"state": "SUCCESS", "result": export_pk}) async_result.return_value = job - get_data = {'job_uuid': task_id} - request = self.factory.get('/', data=get_data, **self.extra) + get_data = {"job_uuid": task_id} + request = self.factory.get("/", data=get_data, **self.extra) response = view(request, pk=self.data_view.pk) - self.assertIn('export_url', response.data) + self.assertIn("export_url", response.data) self.assertTrue(async_result.called) self.assertEqual(response.status_code, 202) export = Export.objects.get(task_id=task_id) self.assertTrue(export.is_successful) - with open(export.full_filepath, encoding='utf-8') as csv_file: + with open(export.full_filepath, encoding="utf-8") as csv_file: self.assertEqual( - csv_file.read(), - 'name,age,gender\nDennis Wambua,28,male\n') + csv_file.read(), "name,age,gender\nDennis Wambua,28,male\n" + ) diff --git a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py index 30d4c794d0..27ec249dfe 100644 --- a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py @@ -105,7 +105,7 @@ def _create_merged_dataset(self, geo=False): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } # anonymous user request = self.factory.post('/', data=data) @@ -584,7 +584,7 @@ def test_md_has_deleted_xforms(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) @@ -614,7 +614,7 @@ def test_md_has_no_matching_fields(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) @@ -688,7 +688,7 @@ def test_xform_has_uncommon_reference(self): 'name': 'Merged Dataset', 'project': - "http://testserver/api/v1/projects/%s" % self.project.pk, + f"http://testserver/api/v1/projects/{self.project.pk}", } request = self.factory.post('/', data=data, **self.extra) diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index 16c0da382d..cb7d19fd9d 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -4,10 +4,10 @@ """ import json import os -from builtins import str + +from collections import OrderedDict from six import iteritems from operator import itemgetter -from collections import OrderedDict from django.conf import settings from django.db.models import Q @@ -60,30 +60,39 @@ ] +# pylint: disable=unused-argument @urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock(url, request): + """Mock Enketo responses""" response = requests.Response() response.status_code = 201 - response._content = ( - '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n' ' "code": "200"\n}' + setattr( + response, + "_content", + ('{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n "code": "200"\n}'), ) + return response def get_latest_tags(project): + """Return given project tags as a list.""" project.refresh_from_db() return [tag.name for tag in project.tags.all()] class TestProjectViewSet(TestAbstractViewSet): + """Test ProjectViewSet.""" + def setUp(self): - super(TestProjectViewSet, self).setUp() + super().setUp() self.view = ProjectViewSet.as_view({"get": "list", "post": "create"}) def tearDown(self): cache.clear() - super(TestProjectViewSet, self).tearDown() + super().tearDown() + # pylint: disable=invalid-name @patch("onadata.apps.main.forms.urlopen") def test_publish_xlsform_using_url_upload(self, mock_urlopen): with HTTMock(enketo_mock): @@ -103,23 +112,23 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file - - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request, pk=project_id) - - mock_urlopen.assert_called_with(xls_url) - xls_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) - self.assertEqual( - XFormVersion.objects.filter( - xform__pk=response.data.get("formid") - ).count(), - 1, - ) + with open(path, "rb") as xls_file: + mock_urlopen.return_value = xls_file + + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, pk=project_id) + + mock_urlopen.assert_called_with(xls_url) + xls_file.close() + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual( + XFormVersion.objects.filter( + xform__pk=response.data.get("formid") + ).count(), + 1, + ) def test_projects_list(self): self._project_create() @@ -133,7 +142,9 @@ def test_projects_list(self): self.assertEqual(response.data, [serializer.data]) self.assertIn("created_by", list(response.data[0])) + # pylint: disable=invalid-name def test_project_list_returns_projects_for_active_users_only(self): + """Test project list returns projects of active users only.""" self._project_create() alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) @@ -177,6 +188,7 @@ def test_project_list_returns_projects_for_active_users_only(self): break self.assertTrue(shared_project_in_response) + # pylint: disable=invalid-name def test_project_list_returns_users_own_project_is_shared_to(self): """ Ensure that the project list responses for project owners @@ -252,6 +264,7 @@ def test_project_get_deleted_form(self): self.assertEqual(len(response.data.get("forms")), 0) self.assertEqual(response.status_code, 200) + # pylint: disable=invalid-name def test_none_empty_forms_and_dataview_properties_in_returned_json(self): self._publish_xls_form_to_project() self._create_dataview() @@ -362,7 +375,7 @@ def test_projects_create(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) - def test_project_create_other_account(self): # pylint: disable=C0103 + def test_project_create_other_account(self): # pylint: disable=invalid-name """ Test that a user cannot create a project in a different user account without the right permission. @@ -384,8 +397,8 @@ def test_project_create_other_account(self): # pylint: disable=C0103 response.data, { "owner": [ - "You do not have permission to create a project in " - "the organization {}.".format(bob) + f"You do not have permission to create a project " + f"in the organization {bob}." ] }, ) @@ -406,6 +419,7 @@ def test_project_create_other_account(self): # pylint: disable=C0103 # But under Bob's account self.assertEqual(bob, project.organization) + # pylint: disable=invalid-name def test_create_duplicate_project(self): """ Test creating a project with the same name @@ -413,7 +427,7 @@ def test_create_duplicate_project(self): # data to create project data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -448,10 +462,11 @@ def test_create_duplicate_project(self): final_count = Project.objects.count() self.assertEqual(after_count, final_count) + # pylint: disable=invalid-name def test_projects_create_no_metadata(self): data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "public": False, } self._project_create(project_data=data, merge=False) @@ -465,6 +480,7 @@ def test_projects_create_no_metadata(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) + # pylint: disable=invalid-name def test_projects_create_many_users(self): self._project_create() alice_data = {"username": "alice", "email": "alice@localhost.com"} @@ -477,6 +493,7 @@ def test_projects_create_many_users(self): self.assertEqual(self.user, project.created_by) self.assertEqual(self.user, project.organization) + # pylint: disable=invalid-name def test_publish_xls_form_to_project(self): self._publish_xls_form_to_project() project_name = "another project" @@ -488,10 +505,10 @@ def test_num_datasets(self): self.project.refresh_from_db() request = self.factory.post("/", data={}, **self.extra) request.user = self.user - self.project_data = ProjectSerializer( + project_data = ProjectSerializer( self.project, context={"request": request} ).data - self.assertEqual(self.project_data["num_datasets"], 1) + self.assertEqual(project_data["num_datasets"], 1) def test_last_submission_date(self): self._publish_xls_form_to_project() @@ -499,15 +516,13 @@ def test_last_submission_date(self): request = self.factory.post("/", data={}, **self.extra) request.user = self.user self.project.refresh_from_db() - self.project_data = ProjectSerializer( + project_data = ProjectSerializer( self.project, context={"request": request} ).data date_created = self.xform.instances.order_by("-date_created").values_list( "date_created", flat=True )[0] - self.assertEqual( - str(self.project_data["last_submission_date"]), str(date_created) - ) + self.assertEqual(str(project_data["last_submission_date"]), str(date_created)) def test_view_xls_form(self): self._publish_xls_form_to_project() @@ -538,7 +553,7 @@ def test_view_xls_form(self): ("data_file_type", None), ("media_url", None), ("file_hash", None), - ("url", "http://testserver/api/v1/metadata/%s" % url.pk), + ("url", f"http://testserver/api/v1/metadata/{url.pk}"), ("date_created", url.date_created), ] ), @@ -554,7 +569,7 @@ def test_view_xls_form(self): ("file_hash", None), ( "url", - "http://testserver/api/v1/metadata/%s" % preview_url.pk, + "http://testserver/api/v1/metadata/{preview_url.pk}", ), ("date_created", preview_url.date_created), ] @@ -571,8 +586,7 @@ def test_view_xls_form(self): ("file_hash", None), ( "url", - "http://testserver/api/v1/metadata/%s" - % single_submit_url.pk, + "http://testserver/api/v1/metadata/{single_submit_url.pk}", ), ("date_created", single_submit_url.date_created), ] @@ -638,6 +652,7 @@ def test_assign_form_to_project(self): self.assertEqual(len(response.data["forms"]), old_project_form_count - 1) self.assertEqual(response.data["num_datasets"], old_project_num_datasets - 1) + # pylint: disable=invalid-name def test_project_manager_can_assign_form_to_project(self): view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) self._publish_xls_form_to_project() @@ -670,6 +685,7 @@ def test_project_manager_can_assign_form_to_project(self): self.assertIn("forms", list(response.data)) self.assertEqual(len(response.data["forms"]), 1) + # pylint: disable=invalid-name def test_project_manager_can_assign_form_to_project_no_perm(self): # user must have owner/manager permissions view = ProjectViewSet.as_view({"post": "forms", "get": "retrieve"}) @@ -693,6 +709,7 @@ def test_project_manager_can_assign_form_to_project_no_perm(self): response = view(request, pk=project_id) self.assertEqual(response.status_code, 403) + # pylint: disable=invalid-name def test_project_users_get_readonly_role_on_add_form(self): self._project_create() alice_data = {"username": "alice", "email": "alice@localhost.com"} @@ -703,10 +720,12 @@ def test_project_users_get_readonly_role_on_add_form(self): self.assertTrue(ReadOnlyRole.user_has_role(alice_profile.user, self.xform)) self.assertFalse(OwnerRole.user_has_role(alice_profile.user, self.xform)) + # pylint: disable=invalid-name,too-many-locals @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_reject_form_transfer_if_target_account_has_id_string_already( self, mock_send_mail ): + """Test transfer form fails when a form with same id_string exists.""" # create bob's project and publish a form to it self._publish_xls_form_to_project() projectid = self.project.pk @@ -764,7 +783,7 @@ def test_reject_form_transfer_if_target_account_has_id_string_already( request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=alices_project.id) self.assertEqual(response.status_code, 400) - self.assertEquals( + self.assertEqual( response.data.get("detail"), "Form with the same id_string already exists in this account", ) @@ -782,11 +801,12 @@ def test_reject_form_transfer_if_target_account_has_id_string_already( request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=new_project_id) self.assertEqual(response.status_code, 400) - self.assertEquals( + self.assertEqual( response.data.get("detail"), "Form with the same id_string already exists in this account", ) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_allow_form_transfer_if_org_is_owned_by_user(self, mock_send_mail): # create bob's project and publish a form to it @@ -826,6 +846,7 @@ def test_allow_form_transfer_if_org_is_owned_by_user(self, mock_send_mail): bobs_results = response.data self.assertListEqual(bobs_results.get("forms"), []) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): # create bob's project and publish a form to it @@ -860,6 +881,7 @@ def test_handle_integrity_error_on_form_transfer(self, mock_send_mail): "Form with the same id_string already exists in this account", ) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): projects_count = Project.objects.count() @@ -874,7 +896,7 @@ def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): { "name": "alice's project", "owner": ( - "http://testserver/api/v1/users/%s" % alice_profile.user.username + f"http://testserver/api/v1/users/{alice_profile.user.username}" ), "public": False, }, @@ -916,9 +938,7 @@ def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): self._login_user_and_profile(alice_data) self.project = alice_project data = { - "owner": ( - "http://testserver/api/v1/users/%s" % alice_profile.user.username - ), + "owner": ("http://testserver/api/v1/users/{alice_profile.user.username}"), "public": True, "public_data": True, "description": "transportation_2011_07_25", @@ -945,6 +965,7 @@ def test_form_transfer_when_org_creator_creates_project(self, mock_send_mail): response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) + # pylint: disable=invalid-name def test_project_transfer_upgrades_permissions(self): """ Test that existing project permissions are updated when necessary @@ -972,7 +993,7 @@ def test_project_transfer_upgrades_permissions(self): self._login_user_and_profile({"username": bob.username, "email": bob.email}) self._org_create() self.assertEqual(self.organization.created_by, bob) - org_url = "http://testserver/api/v1/users/" f"{self.organization.user.username}" + org_url = f"http://testserver/api/v1/users/{self.organization.user.username}" view = OrganizationProfileViewSet.as_view({"post": "members"}) data = {"username": alice_profile.user.username, "role": OwnerRole.name} request = self.factory.post( @@ -1017,6 +1038,7 @@ def test_project_transfer_upgrades_permissions(self): if owner: self.assertEqual(user["role"], OwnerRole.name) + # pylint: disable=invalid-name @override_settings(ALLOW_PUBLIC_DATASETS=False) def test_disallow_public_project_creation(self): """ @@ -1026,7 +1048,7 @@ def test_disallow_public_project_creation(self): view = ProjectViewSet.as_view({"post": "create"}) data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "public": True, } request = self.factory.post("/", data=data, **self.extra) @@ -1036,6 +1058,7 @@ def test_disallow_public_project_creation(self): response.data["public"][0], "Public projects are currently disabled." ) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_form_transfer_when_org_admin_not_creator_creates_project( self, mock_send_mail @@ -1052,7 +1075,7 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( { "name": "alice's project", "owner": ( - "http://testserver/api/v1/users/%s" % alice_profile.user.username + f"http://testserver/api/v1/users/{alice_profile.user.username}" ), "public": False, }, @@ -1094,9 +1117,7 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( # let alice create a form in her personal project self.project = alice_project data = { - "owner": ( - "http://testserver/api/v1/users/%s" % alice_profile.user.username - ), + "owner": ("http://testserver/api/v1/users/{alice_profile.user.username}"), "public": True, "public_data": True, "description": "transportation_2011_07_25", @@ -1123,6 +1144,7 @@ def test_form_transfer_when_org_admin_not_creator_creates_project( response = view(request, pk=org_project.id) self.assertEqual(response.status_code, 201) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint(self, mock_send_mail): # create project and publish form to project @@ -1160,8 +1182,10 @@ def test_project_share_endpoint(self, mock_send_mail): self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) + # pylint: disable=protected-access role_class._remove_obj_permissions(alice_profile.user, self.project) + # pylint: disable=invalid-name @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_project_share_endpoint_form_published_later(self, mock_send_mail): # create project @@ -1202,6 +1226,7 @@ def test_project_share_endpoint_form_published_later(self, mock_send_mail): self.assertEqual(response.get("Cache-Control"), None) self.assertFalse(mock_send_mail.called) + # pylint: disable=protected-access role_class._remove_obj_permissions(alice_profile.user, self.project) self.xform.delete() @@ -1225,6 +1250,7 @@ def test_project_share_remove_user(self): self.assertEqual(response.status_code, 204) self.assertFalse(role_class.user_has_role(alice_profile.user, self.project)) + # pylint: disable=too-many-statements def test_project_filter_by_owner(self): """ Test projects endpoint filter by owner. @@ -1345,6 +1371,7 @@ def test_project_partial_updates(self): self.assertEqual(response.status_code, 200) self.assertEqual(project.metadata, json_metadata) + # pylint: disable=invalid-name def test_cache_updated_on_project_update(self): view = ProjectViewSet.as_view({"get": "retrieve", "patch": "partial_update"}) self._project_create() @@ -1377,7 +1404,7 @@ def test_project_put_updates(self): projectid = self.project.pk data = { "name": "updated name", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "description", "location": "Nairobi, Kenya", @@ -1390,6 +1417,7 @@ def test_project_put_updates(self): data.update({"metadata": json.loads(data.get("metadata"))}) self.assertDictContainsSubset(data, response.data) + # pylint: disable=invalid-name def test_project_partial_updates_to_existing_metadata(self): self._project_create() view = ProjectViewSet.as_view({"patch": "partial_update"}) @@ -1404,6 +1432,7 @@ def test_project_partial_updates_to_existing_metadata(self): self.assertEqual(response.status_code, 200) self.assertEqual(project.metadata, json_metadata) + # pylint: disable=invalid-name def test_project_update_shared_cascades_to_xforms(self): self._publish_xls_form_to_project() view = ProjectViewSet.as_view({"patch": "partial_update"}) @@ -1431,6 +1460,7 @@ def test_project_add_star(self): self.assertEqual(len(self.project.user_stars.all()), 1) self.assertEqual(self.project.user_stars.all()[0], self.user) + # pylint: disable=invalid-name def test_create_project_invalid_metadata(self): """ Make sure that invalid metadata values are outright rejected @@ -1439,7 +1469,7 @@ def test_create_project_invalid_metadata(self): view = ProjectViewSet.as_view({"post": "create"}) data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": "null", "public": False, } @@ -1494,9 +1524,7 @@ def test_project_get_starred_by(self): self.assertEqual(len(response.data), 2) alice_profile, bob_profile = sorted(response.data, key=itemgetter("username")) - self.assertEquals( - sorted(bob_profile.items()), sorted(user_profile_data.items()) - ) + self.assertEqual(sorted(bob_profile.items()), sorted(user_profile_data.items())) self.assertEqual(alice_profile["username"], "alice") def test_user_can_view_public_projects(self): @@ -1522,7 +1550,7 @@ def test_user_can_view_public_projects(self): def test_projects_same_name_diff_case(self): data1 = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -1539,7 +1567,7 @@ def test_projects_same_name_diff_case(self): data2 = { "name": "DEMO", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -1575,7 +1603,7 @@ def test_projects_get_exception(self): # invalid id response = view(request, pk="1w") self.assertEqual(response.status_code, 400) - error_msg = "Invalid value for project_id. It must be a " "positive integer." + error_msg = "Invalid value for project_id. It must be a positive integer." self.assertEqual(str(response.data["detail"]), error_msg) def test_publish_to_public_project(self): @@ -1591,8 +1619,8 @@ def test_publish_to_public_project(self): self.project = public_project self._publish_xls_form_to_project(public=True) - self.assertEquals(self.xform.shared, True) - self.assertEquals(self.xform.shared_data, True) + self.assertEqual(self.xform.shared, True) + self.assertEqual(self.xform.shared_data, True) def test_public_form_private_project(self): self.project = Project( @@ -1656,8 +1684,7 @@ def test_publish_to_public_project_public_form(self): self.project = public_project data = { - "owner": "http://testserver/api/v1/users/%s" - % self.project.organization.username, + "owner": f"http://testserver/api/v1/users/{self.project.organization.username}", "public": True, "public_data": True, "description": "transportation_2011_07_25", @@ -1671,8 +1698,8 @@ def test_publish_to_public_project_public_form(self): } self._publish_xls_form_to_project(publish_data=data, merge=False) - self.assertEquals(self.xform.shared, True) - self.assertEquals(self.xform.shared_data, True) + self.assertEqual(self.xform.shared, True) + self.assertEqual(self.xform.shared_data, True) def test_project_all_users_can_share_remove_themselves(self): self._publish_xls_form_to_project() @@ -1717,7 +1744,7 @@ def test_owner_cannot_remove_self_if_no_other_owner(self): self.assertEqual(response.status_code, 400) error = {"remove": ["Project requires at least one owner"]} - self.assertEquals(response.data, error) + self.assertEqual(response.data, error) self.assertTrue(OwnerRole.user_has_role(bob_profile.user, self.project)) @@ -1745,12 +1772,12 @@ def test_last_date_modified_changes_when_adding_new_form(self): self.project.refresh_from_db() current_last_date = self.project.date_modified - self.assertNotEquals(last_date, current_last_date) + self.assertNotEqual(last_date, current_last_date) self._make_submissions() self.project.refresh_from_db() - self.assertNotEquals(current_last_date, self.project.date_modified) + self.assertNotEqual(current_last_date, self.project.date_modified) def test_anon_project_form_endpoint(self): self._project_create() @@ -1816,7 +1843,7 @@ def test_move_project_owner(self): view = ProjectViewSet.as_view({"patch": "partial_update"}) - data_patch = {"owner": "http://testserver/api/v1/users/%s" % alice.username} + data_patch = {"owner": f"http://testserver/api/v1/users/{alice.username}"} request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) @@ -1830,7 +1857,7 @@ def test_move_project_owner(self): response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() - self.assertEquals(self.project.organization, alice) + self.assertEqual(self.project.organization, alice) self.assertTrue(OwnerRole.user_has_role(alice, self.project)) def test_cannot_share_project_to_owner(self): @@ -1849,7 +1876,7 @@ def test_cannot_share_project_to_owner(self): self.assertEqual(response.status_code, 400) self.assertEqual( - response.data["username"], ["Cannot share project" " with the owner (bob)"] + response.data["username"], ["Cannot share project with the owner (bob)"] ) self.assertTrue(OwnerRole.user_has_role(self.user, self.project)) @@ -1879,7 +1906,7 @@ def test_project_share_readonly(self): if user == alice_profile.user: r = p.get("role") - self.assertEquals(r, ReadOnlyRole.name) + self.assertEqual(r, ReadOnlyRole.name) def test_move_project_owner_org(self): # create project and publish form to project @@ -1892,8 +1919,7 @@ def test_move_project_owner_org(self): old_org = self.project.organization data_patch = { - "owner": "http://testserver/api/v1/users/%s" - % self.organization.user.username + "owner": f"http://testserver/api/v1/users/{self.organization.user.username}" } request = self.factory.patch("/", data=data_patch, **self.extra) response = view(request, pk=projectid) @@ -2011,11 +2037,11 @@ def test_project_share_readonly_no_downloads(self): for user in users: if user.get("user") == "bob": - self.assertEquals(user.get("role"), "owner") + self.assertEqual(user.get("role"), "owner") elif user.get("user") == "alice": - self.assertEquals(user.get("role"), "readonly-no-download") + self.assertEqual(user.get("role"), "readonly-no-download") elif user.get("user") == "tom": - self.assertEquals(user.get("role"), "readonly") + self.assertEqual(user.get("role"), "readonly") def test_team_users_in_a_project(self): self._team_create() @@ -2050,9 +2076,9 @@ def test_team_users_in_a_project(self): response = view(request, pk=project.pk) self.assertIsNotNone(response.data["teams"]) - self.assertEquals(3, len(response.data["teams"])) - self.assertEquals(response.data["teams"][2]["role"], "editor") - self.assertEquals( + self.assertEqual(3, len(response.data["teams"])) + self.assertEqual(response.data["teams"][2]["role"], "editor") + self.assertEqual( response.data["teams"][2]["users"][0], str(chuck_profile.user.username) ) @@ -2085,7 +2111,7 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): ) response = view(request, user="denoinc") - self.assertEquals(201, response.status_code) + self.assertEqual(201, response.status_code) data = json.dumps( {"username": chuck_profile.user.username, "role": OwnerRole.name} ) @@ -2095,15 +2121,14 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): ) response = view(request, user="denoinc") - self.assertEquals(201, response.status_code) + self.assertEqual(201, response.status_code) # admin 2 creates a project self.user = chuck_profile.user - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" - % self.organization.user.username, + "owner": f"http://testserver/api/v1/users/{self.organization.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -2117,12 +2142,12 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): # admin 1 tries to access project created by admin 2 self.user = alice_profile.user - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) - self.assertEquals(200, response.status_code) + self.assertEqual(200, response.status_code) # assert admin can add colaborators tompoo_data = {"username": "tompoo", "email": "tompoo@localhost.com"} @@ -2137,7 +2162,7 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): self.assertEqual(response.status_code, 204) self.user = bob - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % bob.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {bob.auth_token}"} # remove from admin org data = json.dumps({"username": alice_profile.user.username}) @@ -2147,25 +2172,25 @@ def test_project_accesible_by_admin_created_by_diff_admin(self): "/", data=data, content_type="application/json", **self.extra ) response = view(request, user="denoinc") - self.assertEquals(200, response.status_code) + self.assertEqual(200, response.status_code) view = ProjectViewSet.as_view({"get": "retrieve"}) self.user = alice_profile.user - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) # user cant access the project removed from org - self.assertEquals(404, response.status_code) + self.assertEqual(404, response.status_code) def test_public_project_on_creation(self): view = ProjectViewSet.as_view({"post": "create"}) data = { "name": "demopublic", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -2190,7 +2215,7 @@ def test_permission_passed_to_dataview_parent_form(self): self._publish_xls_form_to_project() data = { "name": "demo2", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -2205,8 +2230,8 @@ def test_permission_passed_to_dataview_parent_form(self): data = { "name": "My DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{project2.pk}", "columns": columns, "query": "[ ]", } @@ -2243,7 +2268,7 @@ def test_permission_not_passed_to_dataview_parent_form(self): self._publish_xls_form_to_project() data = { "name": "demo2", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -2256,8 +2281,8 @@ def test_permission_not_passed_to_dataview_parent_form(self): data = { "name": "My DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % project2.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{project2.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"20"},' '{"column":"age","filter":"<","value":"50"}]', @@ -2498,9 +2523,9 @@ def test_project_share_multiple_users(self): for user in users: if user.get("user") == "bob": - self.assertEquals(user.get("role"), "owner") + self.assertEqual(user.get("role"), "owner") else: - self.assertEquals(user.get("role"), "readonly") + self.assertEqual(user.get("role"), "readonly") @patch("onadata.apps.api.viewsets.project_viewset.send_mail") def test_sends_mail_on_multi_share(self, mock_send_mail): diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index ef921c6bf3..b6e5427cae 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -4346,8 +4346,8 @@ def test_xform_linked_dataviews(self): data = { "name": "Another DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % self.project.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"50"}]', } @@ -4397,8 +4397,8 @@ def test_delete_xform_also_deletes_linked_dataviews(self): self._create_dataview() data = { "name": "Another DataView", - "xform": "http://testserver/api/v1/forms/%s" % self.xform.pk, - "project": "http://testserver/api/v1/projects/%s" % self.project.pk, + "xform": f"http://testserver/api/v1/forms/{self.xform.pk}", + "project": f"http://testserver/api/v1/projects/{self.project.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"50"}]', } From e025903f9dfebfb06e232b2442fcd479097697fc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 12:38:29 +0300 Subject: [PATCH 059/234] Fix f-string syntax --- .../apps/api/tests/viewsets/test_xform_viewset.py | 12 ++++++------ onadata/apps/logger/models/xform.py | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b6e5427cae..b254b80b94 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -1797,17 +1797,17 @@ def test_form_add_project_cache(self): self._project_create() # set project XForm cache - cache.set("{}{}".format(PROJ_FORMS_CACHE, self.project.pk), ["forms"]) + cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", ["forms"]) self.assertNotEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None ) self._publish_xls_form_to_project() # test project XForm cache is empty self.assertEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None ) def test_form_delete(self): @@ -1822,10 +1822,10 @@ def test_form_delete(self): self.assertNotEqual(etag_value, None) # set project XForm cache - cache.set("{}{}".format(PROJ_FORMS_CACHE, self.project.pk), ["forms"]) + cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", ["forms"]) self.assertNotEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None ) view = XFormViewSet.as_view({"delete": "destroy", "get": "retrieve"}) @@ -1837,7 +1837,7 @@ def test_form_delete(self): # test project XForm cache is emptied self.assertEqual( - cache.get("{}{}".format(PROJ_FORMS_CACHE, self.project.pk)), None + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None ) self.xform.refresh_from_db() diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 5869705246..530b03b00c 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -891,7 +891,7 @@ def _set_title(self): title_xml = self.title[:XFORM_TITLE_LENGTH] if isinstance(self.xml, b): self.xml = self.xml.decode("utf-8") - self.xml = title_pattern.sub("{title_xml}", self.xml) + self.xml = title_pattern.sub(f"{title_xml}", self.xml) self._set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( @@ -1027,7 +1027,6 @@ def soft_delete(self, user=None): without violating the uniqueness constraint. Also soft deletes associated dataviews """ - soft_deletion_time = timezone.now() deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time @@ -1060,6 +1059,7 @@ def soft_delete(self, user=None): # Delete associated Form Media Files for metadata in self.metadata_set.filter(deleted_at__isnull=True): metadata.soft_delete() + clear_project_cache(self.project_id) def submission_count(self, force_update=False): """Returns the form's number of submission.""" @@ -1171,6 +1171,7 @@ def update_profile_num_submissions(sender, instance, **kwargs): def clear_project_cache(project_id): + """Clear project cache""" safe_delete(f"{PROJ_OWNER_CACHE}{project_id}") safe_delete(f"{PROJ_FORMS_CACHE}{project_id}") safe_delete(f"{PROJ_BASE_FORMS_CACHE}{project_id}") From 6b0280f0aa3c1304df95b7d45191f6c6381f587c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 13:26:27 +0300 Subject: [PATCH 060/234] Fix project caching clearing --- .../tests/viewsets/test_project_viewset.py | 4 +++- .../api/tests/viewsets/test_xform_viewset.py | 23 ++++++++----------- onadata/apps/logger/models/xform.py | 1 + onadata/apps/viewer/models/data_dictionary.py | 3 +-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index cb7d19fd9d..d388c13b22 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -2575,7 +2575,9 @@ def test_project_caching(self): response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data["forms"]), 1) - self.assertEqual(response.data["forms"][0]["name"], self.xform.title) + self.assertEqual( + response.data["forms"][0]["name"], self.xform.title, response.data + ) self.assertEqual( response.data["forms"][0]["last_submission_time"], self.xform.time_of_last_submission(), diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b254b80b94..b47abde86f 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -1796,18 +1796,17 @@ def test_form_add_project_cache(self): with HTTMock(enketo_mock): self._project_create() + cleared_cache_content = ["forms"] # set project XForm cache - cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", ["forms"]) + cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", cleared_cache_content) - self.assertNotEqual( - cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None - ) + self.assertNotEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) self._publish_xls_form_to_project() - # test project XForm cache is empty - self.assertEqual( - cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None + # test project XForm cache has new content + self.assertNotEqual( + cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), cleared_cache_content ) def test_form_delete(self): @@ -1824,9 +1823,7 @@ def test_form_delete(self): # set project XForm cache cache.set(f"{PROJ_FORMS_CACHE}{self.project.pk}", ["forms"]) - self.assertNotEqual( - cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None - ) + self.assertNotEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) view = XFormViewSet.as_view({"delete": "destroy", "get": "retrieve"}) formid = self.xform.pk @@ -1836,9 +1833,7 @@ def test_form_delete(self): self.assertEqual(response.status_code, 204) # test project XForm cache is emptied - self.assertEqual( - cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None - ) + self.assertEqual(cache.get(f"{PROJ_FORMS_CACHE}{self.project.pk}"), None) self.xform.refresh_from_db() @@ -1975,7 +1970,7 @@ def test_form_clone_endpoint(self): self.assertEqual(response.status_code, 400) self.assertEqual( str(response.data["project"]), - '[ErrorDetail(string="Field \'id\' expected a number but got \'abc123\'.", code=\'invalid\')]', + "[ErrorDetail(string=\"Field 'id' expected a number but got 'abc123'.\", code='invalid')]", ) # pylint: disable=no-member diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 530b03b00c..987401268a 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1182,6 +1182,7 @@ def clear_project_cache(project_id): # pylint: disable=unused-argument def save_project(sender, instance=None, created=False, **kwargs): """Update the date_modified field in the XForm's project.""" + clear_project_cache(instance.project_id) instance.project.save(update_fields=["date_modified"]) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index cb5c3c9c99..8b5942fed3 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -7,7 +7,6 @@ import unicodecsv as csv import openpyxl -from builtins import str as text from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone @@ -37,7 +36,7 @@ def is_newline_error(e): "new-line character seen in unquoted field - do you need" " to open the file in universal-newline mode?" ) - return newline_error == text(e) + return newline_error == str(e) def process_xlsform(xls, default_name): From 30f960efca226fa595552c5cc6757469d8ab7e4f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 13:34:21 +0300 Subject: [PATCH 061/234] Fix test check string --- onadata/apps/api/tests/viewsets/test_dataview_viewset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index 8b8b7fd7ee..84305423b3 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -544,8 +544,9 @@ def test_dataview_sql_injection(self): self.assertTrue( str(response.data.get("detail")).startswith( - "invalid input syntax for integer" - ) + "invalid input syntax for type integer" + ), + response.data, ) def test_dataview_invalid_columns(self): From 10e7261a9de6a398a278867c435deb90f4aef2cc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 13:42:57 +0300 Subject: [PATCH 062/234] Fix generate_qrcode: Rename format to image_format Addresses pylint: redefined-builtin / Redefining built-in 'format' --- onadata/libs/utils/qrcode.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/qrcode.py b/onadata/libs/utils/qrcode.py index f82673b917..9a104e55e0 100644 --- a/onadata/libs/utils/qrcode.py +++ b/onadata/libs/utils/qrcode.py @@ -7,13 +7,14 @@ from io import BytesIO +# pylint: disable=too-many-arguments def generate_qrcode( message, stream=None, eclevel="M", margin=10, data_mode="8bits", - format="PNG", + image_format="PNG", scale=2.5, ): """Generate a QRCode, settings options and output.""" @@ -33,9 +34,9 @@ def generate_qrcode( scale=scale, ) - img.save(stream, format) + img.save(stream, image_format) - datauri = "data:image/png;base64,%s" % b64encode(stream.getvalue()).decode("utf-8") + datauri = f"data:image/png;base64,{b64encode(stream.getvalue()).decode('utf-8')}" stream.close() return datauri From 6239e3016fc321fa7f9db4cb07d73a4a3a9bb2b6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 15:58:21 +0300 Subject: [PATCH 063/234] Cleanup --- .../libs/tests/utils/test_export_builder.py | 1149 ++++++++--------- 1 file changed, 574 insertions(+), 575 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 1f67fce10e..0ec03bd342 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -10,7 +10,6 @@ import shutil import tempfile import zipfile -from builtins import open from collections import OrderedDict from ctypes import ArgumentError from io import BytesIO @@ -51,6 +50,8 @@ def _logger_fixture_path(*args): class TestExportBuilder(TestBase): + """Test onadata.libs.utils.export_builder functions.""" + data = [ { "name": "Abe", @@ -222,10 +223,11 @@ def _create_childrens_survey(self, filename="childrens_survey.xlsx"): survey = create_survey_from_xls( _logger_fixture_path(filename), default_name=filename.split(".")[0] ) - self.dd = DataDictionary() - self.dd._survey = survey + self.data_dictionary = DataDictionary() + setattr(self.data_dictionary, "_survey", survey) return survey + # pylint: disable=invalid-name def test_build_sections_for_multilanguage_form(self): survey = create_survey_from_xls( _logger_fixture_path("multi_lingual_form.xlsx"), @@ -362,18 +364,17 @@ def test_build_sections_from_survey(self): element_names = [element["xpath"] for element in section["elements"]] self.assertEqual(sorted(expected_element_names), sorted(element_names)) + # pylint: disable=too-many-locals def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -387,34 +388,30 @@ def test_zipped_csv_export_works(self): index += 1 # check that each file exists - self.assertTrue( - os.path.exists(os.path.join(temp_dir, "{0}.csv".format(survey.name))) - ) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.csv"))) with open( - os.path.join(temp_dir, "{0}.csv".format(survey.name)), encoding="utf-8" + os.path.join(temp_dir, f"{survey.name}.csv"), encoding="utf-8" ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file with open( _logger_fixture_path("csvs", "childrens_survey.csv"), encoding="utf-8" ) as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv"), encoding="utf-8") as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file with open( _logger_fixture_path("csvs", "children.csv"), encoding="utf-8" ) as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) self.assertTrue(os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) @@ -422,14 +419,13 @@ def test_zipped_csv_export_works(self): os.path.join(temp_dir, "children_cartoons.csv"), encoding="utf-8" ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file with open( _logger_fixture_path("csvs", "children_cartoons.csv"), encoding="utf-8" ) as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) self.assertTrue( @@ -439,15 +435,14 @@ def test_zipped_csv_export_works(self): os.path.join(temp_dir, "children_cartoons_characters.csv"), encoding="utf-8" ) as csv_file: reader = csv.reader(csv_file) - rows = [r for r in reader] + rows = list(reader) # open comparison file with open( _logger_fixture_path("csvs", "children_cartoons_characters.csv"), encoding="utf-8", ) as fixture_csv: - fixture_reader = csv.reader(fixture_csv) - expected_rows = [r for r in fixture_reader] + expected_rows = list(csv.reader(fixture_csv)) self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir) @@ -460,14 +455,13 @@ def test_xls_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.osm_data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - osm_data_sheet = wb["osm"] - rows = [row for row in osm_data_sheet.rows] - xls_headers = [a.value for a in rows[0]] - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.osm_data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + osm_data_sheet = workbook["osm"] + rows = list(osm_data_sheet.rows) + xls_headers = [a.value for a in rows[0]] expected_column_headers = [ "photo", @@ -510,6 +504,7 @@ def test_xls_export_with_osm_data(self): self.assertEqual(submission[11], "23.707316084046038") self.assertEqual(submission[13], "kol") + # pylint: disable=invalid-name def test_decode_mongo_encoded_section_names(self): data = { "main_section": [1, 2, 3, 4], @@ -524,6 +519,7 @@ def test_decode_mongo_encoded_section_names(self): } self.assertEqual(result, expected_result) + # pylint: disable=invalid-name def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii @@ -534,14 +530,12 @@ def test_zipped_csv_export_works_with_unicode(self): ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents @@ -572,8 +566,8 @@ def test_zipped_csv_export_works_with_unicode(self): "_duration", "_submitted_by", ] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) self.assertEqual(data["children.info/fav_colors/red's"], "True") @@ -581,6 +575,7 @@ def test_zipped_csv_export_works_with_unicode(self): self.assertEqual(data["children.info/fav_colors/pink's"], "False") # check that red and blue are set to true + # pylint: disable=invalid-name def test_zipped_sav_export_with_date_field(self): md = """ | survey | @@ -603,20 +598,18 @@ def test_zipped_sav_export_with_date_field(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[0][0], b"expense_date") self.assertEqual(rows[1][0], b"2013-01-03") @@ -627,6 +620,7 @@ def test_zipped_sav_export_with_date_field(self): shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_sav_export_dynamic_select_multiple(self): md = """ | survey | @@ -658,19 +652,17 @@ def test_zipped_sav_export_dynamic_select_multiple(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[0][0], b"sex") self.assertEqual(rows[1][0], b"male") @@ -693,6 +685,7 @@ def test_zipped_sav_export_dynamic_select_multiple(self): shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_sav_export_with_zero_padded_select_one_field(self): md = """ | survey | @@ -708,24 +701,23 @@ def test_zipped_sav_export_with_zero_padded_select_one_field(self): data = [{"expensed": "09", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0].decode("utf-8"), "09") self.assertEqual(rows[1][4].decode("utf-8"), "2016-11-21 03:43:43") + # pylint: disable=invalid-name def test_zipped_sav_export_with_numeric_select_one_field(self): md = """ | survey | @@ -750,20 +742,18 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) # expensed 1 @@ -778,6 +768,7 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): self.assertEqual(rows[0][5], b"@_submission_time") self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_field_different_groups(self): """ Test SAV exports duplicate fields, same group - one field in repeat @@ -812,21 +803,23 @@ def test_zipped_sav_export_with_duplicate_field_different_groups(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) + # pylint: disable=protected-access labels = export_builder._get_sav_value_labels({"A/allaite": "allaite"}) self.assertEqual(labels, {"allaite": {"1": "Oui", "2": "Non"}}) + # pylint: disable=protected-access repeat_group_labels = export_builder._get_sav_value_labels( {"A/rep/allaite": "allaite"} ) self.assertEqual(repeat_group_labels, {"allaite": {1: "Yes", 2: "No"}}) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - - try: - export_builder.to_zipped_sav(temp_zip_file.name, []) - except ArgumentError as e: - self.fail(e) + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + try: + export_builder.to_zipped_sav(temp_zip_file.name, []) + except ArgumentError as e: + self.fail(e) + # pylint: disable=invalid-name def test_split_select_multiples_choices_with_randomize_param(self): """ Test that `_get_select_mulitples_choices` function generates @@ -862,6 +855,7 @@ def test_split_select_multiples_choices_with_randomize_param(self): """ # noqa: E501 survey = self.md_to_pyxform_survey(md_xform, {"name": "data"}) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True @@ -872,6 +866,7 @@ def test_split_select_multiples_choices_with_randomize_param(self): for e in dd.get_survey_elements_with_choices() if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ][0] + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) @@ -912,6 +907,7 @@ def test_split_select_multiples_choices_with_randomize_param(self): ] self.assertEqual(choices, expected_choices) + # pylint: disable=invalid-name def test_zipped_sav_export_with_numeric_select_multiple_field(self): md = """ | survey | | | | | @@ -939,20 +935,18 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[0][0], b"expensed") @@ -987,6 +981,7 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): md = """ | survey | | | | @@ -1002,20 +997,18 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): data = [{"expensed": "1", "_submission_time": "2016-11-21T03:43:43.000-08:00"}] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS @@ -1026,6 +1019,7 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_sav_export_with_values_split_select_multiple(self): md = """ | survey | | | | @@ -1044,20 +1038,18 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): export_builder = ExportBuilder() export_builder.VALUE_SELECT_MULTIPLES = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # check file's contents with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) self.assertTrue(len(rows) > 1) self.assertEqual(rows[1][0], b"2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS @@ -1068,6 +1060,7 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): md = """ | survey | | | | @@ -1097,18 +1090,17 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) - def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 + # pylint: disable=invalid-name + def test_zipped_sav_export_external_choices(self): # pylint: disable=invalid-name """ Test that an SPSS export does not fail when it has choices from a file. """ @@ -1124,17 +1116,16 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=C0103 ] export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) + # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_column_name(self): """ Test that SAV exports with duplicate column names @@ -1156,14 +1147,12 @@ def test_zipped_sav_export_with_duplicate_column_name(self): export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "sports.sav"))) # check file's contents @@ -1171,7 +1160,7 @@ def test_zipped_sav_export_with_duplicate_column_name(self): with SavReader( os.path.join(temp_dir, "sports.sav"), returnHeader=True ) as reader: - rows = [r for r in reader] + rows = list(reader) # Check that columns are present self.assertIn(b"Sport", rows[0]) @@ -1179,6 +1168,7 @@ def test_zipped_sav_export_with_duplicate_column_name(self): # because rows contains 'sport@d4b6' self.assertIn(b"sport", [x[0:5] for x in rows[0]]) + # pylint: disable=invalid-name def test_xls_export_works_with_unicode(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_unicode.xlsx"), @@ -1186,18 +1176,18 @@ def test_xls_export_works_with_unicode(self): ) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - data = dict([(x.value, y.value) for x, y in children_sheet.columns]) - self.assertTrue(data["children.info/fav_colors/red's"]) - self.assertTrue(data["children.info/fav_colors/blue's"]) - self.assertFalse(data["children.info/fav_colors/pink's"]) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + data = {x.value: y.value for x, y in children_sheet.columns} + self.assertTrue(data["children.info/fav_colors/red's"]) + self.assertTrue(data["children.info/fav_colors/blue's"]) + self.assertFalse(data["children.info/fav_colors/pink's"]) + + # pylint: disable=invalid-name def test_xls_export_with_hxl_adds_extra_row(self): # hxl_example.xlsx contains `instance::hxl` column whose value is #age xlsform_path = os.path.join( @@ -1231,15 +1221,16 @@ def test_xls_export_with_hxl_adds_extra_row(self): temp_xls_file.name, self.data_utf8, columns_with_hxl=columns_with_hxl ) temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["hxl_example"] + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["hxl_example"] self.assertTrue(children_sheet) # we pick the second row because the first row has xform fieldnames - rows = [row for row in children_sheet.rows] + rows = list(children_sheet.rows) hxl_row = [a.value for a in rows[1]] self.assertIn("#age", hxl_row) + # pylint: disable=invalid-name,too-many-locals def test_export_with_image_attachments(self): """ Test that the url for images is displayed correctly in exports @@ -1252,21 +1243,18 @@ def test_export_with_image_attachments(self): self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) - xml_string = """ - + xml_string = f""" + uuid:UJ6jSMAJ1Jz4EszdgHy8n852AsKaqBPO5 1300221157303.jpg - """.format( - self.xform.id_string - ) + """ file_path = ( - "{}/apps/logger/tests/Health_2011_03_13." - "xml_2011-03-15_20-30-28/1300221157303" - ".jpg".format(settings.PROJECT_ROOT) + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303.jpg" ) media_file = django_file( path=file_path, field_name="image1", content_type="image/jpeg" @@ -1281,22 +1269,20 @@ def test_export_with_image_attachments(self): survey = self.md_to_pyxform_survey(md, {"name": "exp"}) export_builder = ExportBuilder() export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file, xdata) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file) - children_sheet = wb["exp"] - self.assertTrue(children_sheet) - rows = [row for row in children_sheet.rows] - row = [a.value for a in rows[1]] - attachment_id = xdata[0]["_attachments"][0]["id"] - attachment_filename = xdata[0]["_attachments"][0]["filename"] - attachment_url = "http://example.com/api/v1/files/{}?filename={}".format( - attachment_id, attachment_filename - ) # noqa - self.assertIn(attachment_url, row) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file, xdata) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file) + children_sheet = workbook["exp"] + self.assertTrue(children_sheet) + rows = list(children_sheet.rows) + row = [a.value for a in rows[1]] + attachment_id = xdata[0]["_attachments"][0]["id"] + attachment_filename = xdata[0]["_attachments"][0]["filename"] + attachment_url = f"http://example.com/api/v1/files/{attachment_id}?filename={attachment_filename}" # noqa + self.assertIn(attachment_url, row) + + # pylint: disable=invalid-name def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1338,6 +1324,7 @@ def test_generation_of_multi_selects_works(self): sorted(expected_select_multiples["children"]["children/ice.creams"]), ) + # pylint: disable=invalid-name def test_split_select_multiples_works(self): """ Test split_select_multiples works as expected. @@ -1387,6 +1374,7 @@ def test_split_select_multiples_works(self): } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_select_mutliples_works_with_int_value_in_row(self): select_multiples = { "children/fav_number": [ @@ -1414,6 +1402,7 @@ def test_split_select_mutliples_works_with_int_value_in_row(self): self.assertTrue(new_row) self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_select_multiples_works_when_data_is_blank(self): select_multiples = { "children/fav_colors": [ @@ -1443,6 +1432,7 @@ def test_split_select_multiples_works_when_data_is_blank(self): } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1486,6 +1476,7 @@ def test_split_gps_components_works(self): } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_split_gps_components_works_when_gps_data_is_blank(self): gps_fields = { "geo/geolocation": [ @@ -1504,14 +1495,15 @@ def test_split_gps_components_works_when_gps_data_is_blank(self): } self.assertEqual(new_row, expected_row) + # pylint: disable=invalid-name def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields = { "childrens_survey": { - "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")), - "tel/tel.mobile": "tel/{0}".format(_encode_for_mongo("tel.mobile")), + "tel/tel.office": f"tel/{_encode_for_mongo('tel.office')}", + "tel/tel.mobile": f"tel/{_encode_for_mongo('tel.mobile')}", } } encoded_fields = export_builder.encoded_fields @@ -1521,14 +1513,13 @@ def test_generation_of_mongo_encoded_fields_works(self): expected_encoded_fields["childrens_survey"], ) + # pylint: disable=invalid-name def test_decode_fields_names_encoded_for_mongo(self): - encoded_fields = { - "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")) - } + encoded_fields = {"tel/tel.office": f"tel/{_encode_for_mongo('tel.office')}"} row = { "name": "Abe", "age": 35, - "tel/{0}".format(_encode_for_mongo("tel.office")): "123-456-789", + f"tel/{_encode_for_mongo('tel.office')}": "123-456-789", } new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields) expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"} @@ -1537,11 +1528,12 @@ def test_decode_fields_names_encoded_for_mongo(self): def test_generate_field_title(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( - "children/age", ".", data_dictionary=self.dd + "children/age", ".", data_dictionary=self.data_dictionary ) expected_field_name = "children.age" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_delimiter_replacement_works_existing_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1559,6 +1551,7 @@ def test_delimiter_replacement_works_existing_fields(self): expected_sections[0]["elements"][0]["title"], ) + # pylint: disable=invalid-name def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1578,6 +1571,7 @@ def test_delimiter_replacement_works_generated_multi_select_fields(self): ][0] self.assertEqual(expected_section["elements"][0]["title"], match["title"]) + # pylint: disable=invalid-name def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1604,156 +1598,167 @@ def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix=".xlsx") - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - # check that we have childrens_survey, children, children_cartoons - # and children_cartoons_characters sheets - expected_sheet_names = [ - "childrens_survey", - "children", - "children_cartoons", - "children_cartoons_characters", - ] - self.assertEqual(list(wb.get_sheet_names()), expected_sheet_names) - - # check header columns - main_sheet = wb.get_sheet_by_name("childrens_survey") - expected_column_headers = [ - "name", - "age", - "geo/geolocation", - "geo/_geolocation_latitude", - "geo/_geolocation_longitude", - "geo/_geolocation_altitude", - "geo/_geolocation_precision", - "tel/tel.office", - "tel/tel.mobile", - "_id", - "meta/instanceID", - "_uuid", - "_submission_time", - "_index", - "_parent_index", - "_parent_table_name", - "_tags", - "_notes", - "_version", - "_duration", - "_submitted_by", - ] - column_headers = list(main_sheet.values)[0] - self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + # check that we have childrens_survey, children, children_cartoons + # and children_cartoons_characters sheets + expected_sheet_names = [ + "childrens_survey", + "children", + "children_cartoons", + "children_cartoons_characters", + ] + self.assertEqual(list(workbook.get_sheet_names()), expected_sheet_names) - childrens_sheet = wb.get_sheet_by_name("children") - expected_column_headers = [ - "children/name", - "children/age", - "children/fav_colors", - "children/fav_colors/red", - "children/fav_colors/blue", - "children/fav_colors/pink", - "children/ice.creams", - "children/ice.creams/vanilla", - "children/ice.creams/strawberry", - "children/ice.creams/chocolate", - "_id", - "_uuid", - "_submission_time", - "_index", - "_parent_index", - "_parent_table_name", - "_tags", - "_notes", - "_version", - "_duration", - "_submitted_by", - ] - column_headers = list(childrens_sheet.values)[0] - self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) + # check header columns + main_sheet = workbook.get_sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo/geolocation", + "geo/_geolocation_latitude", + "geo/_geolocation_longitude", + "geo/_geolocation_altitude", + "geo/_geolocation_precision", + "tel/tel.office", + "tel/tel.mobile", + "_id", + "meta/instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(main_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) - cartoons_sheet = wb.get_sheet_by_name("children_cartoons") - expected_column_headers = [ - "children/cartoons/name", - "children/cartoons/why", - "_id", - "_uuid", - "_submission_time", - "_index", - "_parent_index", - "_parent_table_name", - "_tags", - "_notes", - "_version", - "_duration", - "_submitted_by", - ] - column_headers = list(cartoons_sheet.values)[0] - self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) + childrens_sheet = workbook.get_sheet_by_name("children") + expected_column_headers = [ + "children/name", + "children/age", + "children/fav_colors", + "children/fav_colors/red", + "children/fav_colors/blue", + "children/fav_colors/pink", + "children/ice.creams", + "children/ice.creams/vanilla", + "children/ice.creams/strawberry", + "children/ice.creams/chocolate", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(childrens_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) - characters_sheet = wb.get_sheet_by_name("children_cartoons_characters") - expected_column_headers = [ - "children/cartoons/characters/name", - "children/cartoons/characters/good_or_evil", - "_id", - "_uuid", - "_submission_time", - "_index", - "_parent_index", - "_parent_table_name", - "_tags", - "_notes", - "_version", - "_duration", - "_submitted_by", - ] - column_headers = list(characters_sheet.values)[0] - self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) + cartoons_sheet = workbook.get_sheet_by_name("children_cartoons") + expected_column_headers = [ + "children/cartoons/name", + "children/cartoons/why", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(cartoons_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) - xls_file.close() + characters_sheet = workbook.get_sheet_by_name( + "children_cartoons_characters" + ) + expected_column_headers = [ + "children/cartoons/characters/name", + "children/cartoons/characters/good_or_evil", + "_id", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(characters_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + # pylint: disable=invalid-name def test_to_xls_export_respects_custom_field_delimiter(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix=".xlsx") - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - - # check header columns - main_sheet = wb.get_sheet_by_name("childrens_survey") - expected_column_headers = [ - "name", - "age", - "geo.geolocation", - "geo._geolocation_latitude", - "geo._geolocation_longitude", - "geo._geolocation_altitude", - "geo._geolocation_precision", - "tel.tel.office", - "tel.tel.mobile", - "_id", - "meta.instanceID", - "_uuid", - "_submission_time", - "_index", - "_parent_index", - "_parent_table_name", - "_tags", - "_notes", - "_version", - "_duration", - "_submitted_by", - ] - column_headers = list(main_sheet.values)[0] - self.assertEqual(sorted(list(column_headers)), sorted(expected_column_headers)) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + + # check header columns + main_sheet = workbook.get_sheet_by_name("childrens_survey") + expected_column_headers = [ + "name", + "age", + "geo.geolocation", + "geo._geolocation_latitude", + "geo._geolocation_longitude", + "geo._geolocation_altitude", + "geo._geolocation_precision", + "tel.tel.office", + "tel.tel.mobile", + "_id", + "meta.instanceID", + "_uuid", + "_submission_time", + "_index", + "_parent_index", + "_parent_table_name", + "_tags", + "_notes", + "_version", + "_duration", + "_submitted_by", + ] + column_headers = list(main_sheet.values)[0] + self.assertEqual( + sorted(list(column_headers)), sorted(expected_column_headers) + ) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_duplicates(self): work_sheets = {"childrens_survey": "Worksheet"} desired_sheet_name = "childrens_survey" @@ -1763,6 +1768,7 @@ def test_get_valid_sheet_name_catches_duplicates(self): ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_long_names(self): desired_sheet_name = "childrens_survey_with_a_very_long_name" expected_sheet_name = "childrens_survey_with_a_very_lo" @@ -1771,6 +1777,7 @@ def test_get_valid_sheet_name_catches_long_names(self): ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_get_valid_sheet_name_catches_long_duplicate_names(self): work_sheet_titles = ["childrens_survey_with_a_very_lo"] desired_sheet_name = "childrens_survey_with_a_very_long_name" @@ -1780,6 +1787,7 @@ def test_get_valid_sheet_name_catches_long_duplicate_names(self): ) self.assertEqual(generated_sheet_name, expected_sheet_name) + # pylint: disable=invalid-name def test_to_xls_export_generates_valid_sheet_names(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), @@ -1787,22 +1795,22 @@ def test_to_xls_export_generates_valid_sheet_names(self): ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix=".xlsx") - filename = xls_file.name - export_builder.to_xls_export(filename, self.data) - xls_file.seek(0) - wb = load_workbook(filename) - # check that we have childrens_survey, children, children_cartoons - # and children_cartoons_characters sheets - expected_sheet_names = [ - "childrens_survey_with_a_very_lo", - "childrens_survey_with_a_very_l1", - "childrens_survey_with_a_very_l2", - "childrens_survey_with_a_very_l3", - ] - self.assertEqual(list(wb.get_sheet_names()), expected_sheet_names) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.data) + xls_file.seek(0) + workbook = load_workbook(filename) + # check that we have childrens_survey, children, children_cartoons + # and children_cartoons_characters sheets + expected_sheet_names = [ + "childrens_survey_with_a_very_lo", + "childrens_survey_with_a_very_l1", + "childrens_survey_with_a_very_l2", + "childrens_survey_with_a_very_l3", + ] + self.assertEqual(list(workbook.get_sheet_names()), expected_sheet_names) + # pylint: disable=invalid-name def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_with_a_very_long_name.xlsx"), @@ -1810,26 +1818,25 @@ def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): ) export_builder = ExportBuilder() export_builder.set_survey(survey) - xls_file = NamedTemporaryFile(suffix=".xlsx") - filename = xls_file.name - export_builder.to_xls_export(filename, self.long_survey_data) - xls_file.seek(0) - wb = load_workbook(filename) - - # get the children's sheet - ws1 = wb["childrens_survey_with_a_very_l1"] - - # parent_table is in cell K2 - parent_table_name = ws1["K2"].value - expected_parent_table_name = "childrens_survey_with_a_very_lo" - self.assertEqual(parent_table_name, expected_parent_table_name) - - # get cartoons sheet - ws2 = wb["childrens_survey_with_a_very_l2"] - parent_table_name = ws2["G2"].value - expected_parent_table_name = "childrens_survey_with_a_very_l1" - self.assertEqual(parent_table_name, expected_parent_table_name) - xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as xls_file: + filename = xls_file.name + export_builder.to_xls_export(filename, self.long_survey_data) + xls_file.seek(0) + workbook = load_workbook(filename) + + # get the children's sheet + ws1 = workbook["childrens_survey_with_a_very_l1"] + + # parent_table is in cell K2 + parent_table_name = ws1["K2"].value + expected_parent_table_name = "childrens_survey_with_a_very_lo" + self.assertEqual(parent_table_name, expected_parent_table_name) + + # get cartoons sheet + ws2 = workbook["childrens_survey_with_a_very_l2"] + parent_table_name = ws2["G2"].value + expected_parent_table_name = "childrens_survey_with_a_very_l1" + self.assertEqual(parent_table_name, expected_parent_table_name) def test_type_conversion(self): submission_1 = { @@ -1895,6 +1902,7 @@ def test_type_conversion(self): self.assertIsInstance(new_row["amount"], str) self.assertEqual(new_row["amount"], "") + # pylint: disable=invalid-name def test_xls_convert_dates_before_1900(self): survey = create_survey_from_xls( viewer_fixture_path("test_data_types/test_data_types.xlsx"), @@ -1909,9 +1917,8 @@ def test_xls_convert_dates_before_1900(self): } ] # create export file - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, data) - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, data) # this should error if there is a problem, not sure what to assert def test_convert_types(self): @@ -1938,15 +1945,13 @@ def test_to_sav_export(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -1960,25 +1965,23 @@ def test_to_sav_export(self): index += 1 # check that each file exists - self.assertTrue( - os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) - ) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): with SavReader( - os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + os.path.join(temp_dir, f"{section}.sav"), returnHeader=True ) as reader: header = next(reader) - rows = [r for r in reader] + rows = list(reader) # open comparison file with SavReader( - _logger_fixture_path("spss", "{0}.sav".format(section)), + _logger_fixture_path("spss", f"{section}.sav"), returnHeader=True, ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) if section == "children_cartoons_charactors": @@ -1995,15 +1998,13 @@ def test_to_sav_export_language(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 @@ -2017,27 +2018,25 @@ def test_to_sav_export_language(self): index += 1 # check that each file exists - self.assertTrue( - os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) - ) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): with SavReader( - os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True + os.path.join(temp_dir, f"{section}.sav"), returnHeader=True ) as reader: header = next(reader) - rows = [r for r in reader] + rows = list(reader) if section != "childrens_survey_sw": section += "_sw" # open comparison file with SavReader( - _logger_fixture_path("spss", "{0}.sav".format(section)), + _logger_fixture_path("spss", f"{section}.sav"), returnHeader=True, ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) if section == "children_cartoons_charactors": @@ -2049,25 +2048,31 @@ def _test_sav_file(section): section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + # pylint: disable=invalid-name def test_generate_field_title_truncated_titles(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( - "children/age", "/", data_dictionary=self.dd, remove_group_name=True + "children/age", + "/", + data_dictionary=self.data_dictionary, + remove_group_name=True, ) expected_field_name = "age" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_generate_field_title_truncated_titles_select_multiple(self): self._create_childrens_survey() field_name = ExportBuilder.format_field_title( "children/fav_colors/red", "/", - data_dictionary=self.dd, + data_dictionary=self.data_dictionary, remove_group_name=True, ) expected_field_name = "fav_colors/red" self.assertEqual(field_name, expected_field_name) + # pylint: disable=invalid-name def test_xls_export_remove_group_name(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_unicode.xlsx"), @@ -2076,17 +2081,16 @@ def test_xls_export_remove_group_name(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertTrue(data["fav_colors/red's"]) - self.assertTrue(data["fav_colors/blue's"]) - self.assertFalse(data["fav_colors/pink's"]) - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + data = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) def test_zipped_csv_export_remove_group_name(self): """ @@ -2099,18 +2103,18 @@ def test_zipped_csv_export_remove_group_name(self): export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents - with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: + with open( + os.path.join(temp_dir, "children.info.csv", encoding="utf-8") + ) as csv_file: reader = csv.reader(csv_file) expected_headers = [ "name.first", @@ -2135,8 +2139,8 @@ def test_zipped_csv_export_remove_group_name(self): "_duration", "_submitted_by", ] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) self.assertEqual(data["fav_colors/red's"], "True") @@ -2154,27 +2158,27 @@ def test_xls_export_with_labels(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) - temp_xls_file.seek(0) - # check that values for red\'s and blue\'s are set to true - wb = load_workbook(temp_xls_file.name) - children_sheet = wb["children.info"] - labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(labels["name.first"], "3.1 Childs name") - self.assertEqual(labels["age"], "3.2 Child age") - self.assertEqual(labels["fav_colors/red's"], "fav_colors/Red") - self.assertEqual(labels["fav_colors/blue's"], "fav_colors/Blue") - self.assertEqual(labels["fav_colors/pink's"], "fav_colors/Pink") - - data = dict([(r[0].value, r[2].value) for r in children_sheet.columns]) - self.assertEqual(data["name.first"], "Mike") - self.assertEqual(data["age"], 5) - self.assertTrue(data["fav_colors/red's"]) - self.assertTrue(data["fav_colors/blue's"]) - self.assertFalse(data["fav_colors/pink's"]) - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) + temp_xls_file.seek(0) + # check that values for red\'s and blue\'s are set to true + workbook = load_workbook(temp_xls_file.name) + children_sheet = workbook["children.info"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["name.first"], "3.1 Childs name") + self.assertEqual(labels["age"], "3.2 Child age") + self.assertEqual(labels["fav_colors/red's"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue's"], "fav_colors/Blue") + self.assertEqual(labels["fav_colors/pink's"], "fav_colors/Pink") + + data = {r[0].value: r[2].value for r in children_sheet.columns} + self.assertEqual(data["name.first"], "Mike") + self.assertEqual(data["age"], 5) + self.assertTrue(data["fav_colors/red's"]) + self.assertTrue(data["fav_colors/blue's"]) + self.assertFalse(data["fav_colors/pink's"]) + + # pylint: disable=invalid-name def test_xls_export_with_labels_only(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_unicode.xlsx"), @@ -2190,7 +2194,7 @@ def test_xls_export_with_labels_only(self): # check that values for red\'s and blue\'s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb["children.info"] - data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) + data = {r[0].value: r[1].value for r in children_sheet.columns} self.assertEqual(data["3.1 Childs name"], "Mike") self.assertEqual(data["3.2 Child age"], 5) self.assertTrue(data["fav_colors/Red"]) @@ -2198,6 +2202,7 @@ def test_xls_export_with_labels_only(self): self.assertFalse(data["fav_colors/Pink"]) temp_xls_file.close() + # pylint: disable=invalid-name def test_zipped_csv_export_with_labels(self): """ cvs writer doesnt handle unicode we we have to encode to ascii @@ -2210,14 +2215,12 @@ def test_zipped_csv_export_with_labels(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # check that the children's file (which has the unicode header) exists self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents @@ -2271,10 +2274,10 @@ def test_zipped_csv_export_with_labels(self): "_duration", "_submitted_by", ] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) - actual_labels = [h for h in rows[1]] + actual_labels = list(rows[1]) self.assertEqual(sorted(actual_labels), sorted(expected_labels)) data = dict(zip(rows[0], rows[2])) self.assertEqual(data["fav_colors/red's"], "True") @@ -2283,9 +2286,10 @@ def test_zipped_csv_export_with_labels(self): # check that red and blue are set to true shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_zipped_csv_export_with_labels_only(self): """ - cvs writer doesnt handle unicode we we have to encode to ascii + csv writer does not handle unicode we we have to encode to ascii """ survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_unicode.xlsx"), @@ -2295,16 +2299,14 @@ def test_zipped_csv_export_with_labels_only(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS_ONLY = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - # check that the children's file (which has the unicode header) exists - self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + # check that the children's file (which has the unicode header) exists + self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open( os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" @@ -2333,8 +2335,8 @@ def test_zipped_csv_export_with_labels_only(self): "_duration", "_submitted_by", ] - rows = [row for row in reader] - actual_headers = [h for h in rows[0]] + rows = list(reader) + actual_headers = list(rows[0]) self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) self.assertEqual(data["fav_colors/Red"], "True") @@ -2350,34 +2352,30 @@ def test_to_sav_export_with_labels(self): export_builder.set_survey(survey) export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - filename = temp_zip_file.name - export_builder.to_zipped_sav(filename, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + filename = temp_zip_file.name + export_builder.to_zipped_sav(filename, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] - for d in self.data: + for item in self.data: outputs.append( - dict_to_joined_export(d, index, indices, survey_name, survey, d) + dict_to_joined_export(item, index, indices, survey_name, survey, item) ) index += 1 # check that each file exists - self.assertTrue( - os.path.exists(os.path.join(temp_dir, "{0}.sav".format(survey.name))) - ) + self.assertTrue(os.path.exists(os.path.join(temp_dir, f"{survey.name}.sav"))) def _test_sav_file(section): - sav_path = os.path.join(temp_dir, "{0}.sav".format(section)) + sav_path = os.path.join(temp_dir, f"{section}.sav") if section == "children_survey": with SavHeaderReader(sav_path) as header: expected_labels = [ @@ -2408,22 +2406,23 @@ def _test_sav_file(section): with SavReader(sav_path, returnHeader=True) as reader: header = next(reader) - rows = [r for r in reader] + rows = list(reader) # open comparison file with SavReader( - _logger_fixture_path("spss", "{0}.sav".format(section)), + _logger_fixture_path("spss", f"{section}.sav"), returnHeader=True, ) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) - expected_rows = [r for r in fixture_reader] + expected_rows = list(fixture_reader) self.assertEqual(rows, expected_rows) for section in export_builder.sections: section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + # pylint: disable=invalid-name def test_xls_export_with_english_labels(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_en.xlsx"), @@ -2435,23 +2434,21 @@ def test_xls_export_with_english_labels(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - childrens_survey_sheet = wb["childrens_survey_en"] - labels = dict( - [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] - ) - self.assertEqual(labels["name"], "1. What is your name?") - self.assertEqual(labels["age"], "2. How old are you?") - - children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(labels["fav_colors/red"], "fav_colors/Red") - self.assertEqual(labels["fav_colors/blue"], "fav_colors/Blue") - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + childrens_survey_sheet = workbook["childrens_survey_en"] + labels = {r[0].value: r[1].value for r in childrens_survey_sheet.columns} + self.assertEqual(labels["name"], "1. What is your name?") + self.assertEqual(labels["age"], "2. How old are you?") + + children_sheet = workbook["children"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["fav_colors/red"], "fav_colors/Red") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Blue") + + # pylint: disable=invalid-name def test_xls_export_with_swahili_labels(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_sw.xlsx"), @@ -2463,23 +2460,21 @@ def test_xls_export_with_swahili_labels(self): export_builder.TRUNCATE_GROUP_TITLE = True export_builder.INCLUDE_LABELS = True export_builder.set_survey(survey) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - childrens_survey_sheet = wb["childrens_survey_sw"] - labels = dict( - [(r[0].value, r[1].value) for r in childrens_survey_sheet.columns] - ) - self.assertEqual(labels["name"], "1. Jina lako ni?") - self.assertEqual(labels["age"], "2. Umri wako ni?") - - children_sheet = wb["children"] - labels = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) - self.assertEqual(labels["fav_colors/red"], "fav_colors/Nyekundu") - self.assertEqual(labels["fav_colors/blue"], "fav_colors/Bluu") - temp_xls_file.close() - + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + childrens_survey_sheet = workbook["childrens_survey_sw"] + labels = {r[0].value: r[1].value for r in childrens_survey_sheet.columns} + self.assertEqual(labels["name"], "1. Jina lako ni?") + self.assertEqual(labels["age"], "2. Umri wako ni?") + + children_sheet = workbook["children"] + labels = {r[0].value: r[1].value for r in children_sheet.columns} + self.assertEqual(labels["fav_colors/red"], "fav_colors/Nyekundu") + self.assertEqual(labels["fav_colors/blue"], "fav_colors/Bluu") + + # pylint: disable=invalid-name def test_csv_export_with_swahili_labels(self): survey = create_survey_from_xls( _logger_fixture_path("childrens_survey_sw.xlsx"), @@ -2488,8 +2483,10 @@ def test_csv_export_with_swahili_labels(self): # default_language is set to swahili self.assertEqual(survey.to_json_dict().get("default_language"), "swahili") dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey ordered_columns = OrderedDict() + # pylint: disable=protected-access CSVDataFrameBuilder._build_ordered_columns(survey, ordered_columns) ordered_columns["children/fav_colors/red"] = None labels = get_labels_from_columns(ordered_columns, dd, "/") @@ -2515,6 +2512,7 @@ def test_select_multiples_choices(self): default_name="childrens_survey_sw", ) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True @@ -2526,6 +2524,7 @@ def test_select_multiples_choices(self): if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ][0] self.assertNotEqual(child.children, []) + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) @@ -2572,11 +2571,13 @@ def test_select_multiples_choices(self): CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples ) + # pylint: disable=invalid-name def test_select_multiples_choices_with_choice_filter(self): survey = create_survey_from_xls( _logger_fixture_path("choice_filter.xlsx"), default_name="choice_filter" ) dd = DataDictionary() + # pylint: disable=protected-access dd._survey = survey export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True @@ -2587,6 +2588,7 @@ def test_select_multiples_choices_with_choice_filter(self): for e in dd.get_survey_elements_with_choices() if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ][0] + # pylint: disable=protected-access choices = export_builder._get_select_mulitples_choices( child, dd, ExportBuilder.GROUP_DELIMITER, ExportBuilder.TRUNCATE_GROUP_TITLE ) @@ -2634,10 +2636,12 @@ def test_select_multiples_choices_with_choice_filter(self): ("county/cameron", "cameron", "Cameron"), ] } + # pylint: disable=protected-access self.assertEqual( CSVDataFrameBuilder._collect_select_multiples(dd), select_multiples ) + # pylint: disable=invalid-name def test_string_to_date_with_xls_validation(self): # test "2016-11-02" val = string_to_date_with_xls_validation("2016-11-02") @@ -2675,6 +2679,7 @@ def _create_osm_survey(self): ) return survey + # pylint: disable=invalid-name def test_zip_csv_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports @@ -2685,18 +2690,16 @@ def test_zip_csv_export_has_submission_review_fields(self): export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + # check file's contents - with open(os.path.join(temp_dir, "osm.csv")) as csv_file: - reader = csv.reader(csv_file) - rows = [row for row in reader] + with open(os.path.join(temp_dir, "osm.csv"), encoding="utf-8") as csv_file: + rows = list(csv.reader(csv_file)) actual_headers = rows[0] self.assertIn(REVIEW_COMMENT, sorted(actual_headers)) self.assertIn(REVIEW_DATE, sorted(actual_headers)) @@ -2708,6 +2711,7 @@ def test_zip_csv_export_has_submission_review_fields(self): # check that red and blue are set to true shutil.rmtree(temp_dir) + # pylint: disable=invalid-name def test_xls_export_has_submission_review_fields(self): """ Test that review comment, status and date fields are in xls exports @@ -2718,15 +2722,14 @@ def test_xls_export_has_submission_review_fields(self): export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_xls_file = NamedTemporaryFile(suffix=".xlsx") - export_builder.to_xls_export(temp_xls_file.name, self.osm_data) - temp_xls_file.seek(0) - wb = load_workbook(temp_xls_file.name) - osm_data_sheet = wb["osm"] - rows = [row for row in osm_data_sheet.rows] - xls_headers = [a.value for a in rows[0]] - xls_data = [a.value for a in rows[1]] - temp_xls_file.close() + with NamedTemporaryFile(suffix=".xlsx") as temp_xls_file: + export_builder.to_xls_export(temp_xls_file.name, self.osm_data) + temp_xls_file.seek(0) + workbook = load_workbook(temp_xls_file.name) + osm_data_sheet = workbook["osm"] + rows = list(osm_data_sheet.rows) + xls_headers = [a.value for a in rows[0]] + xls_data = [a.value for a in rows[1]] self.assertIn(REVIEW_COMMENT, sorted(xls_headers)) self.assertIn(REVIEW_DATE, sorted(xls_headers)) self.assertIn(REVIEW_STATUS, sorted(xls_headers)) @@ -2734,6 +2737,7 @@ def test_xls_export_has_submission_review_fields(self): self.assertEqual(xls_data[30], "Wrong Location") self.assertEqual(xls_data[31], "2021-05-25T02:27:19") + # pylint: disable=invalid-name def test_zipped_sav_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports @@ -2744,17 +2748,15 @@ def test_zipped_sav_has_submission_review_fields(self): export_builder = ExportBuilder() export_builder.INCLUDE_REVIEW = True export_builder.set_survey(survey, xform, include_reviews=True) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, self.osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) expected_column_headers = [ x.encode("utf-8") for x in [ @@ -2797,6 +2799,7 @@ def test_zipped_sav_has_submission_review_fields(self): self.assertEqual(rows[1][30], b"Wrong Location") self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") + # pylint: disable=invalid-name def test_zipped_csv_export_with_osm_data(self): """ Tests that osm data is included in zipped csv export @@ -2805,18 +2808,15 @@ def test_zipped_csv_export_with_osm_data(self): xform = self.xform export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() - - with open(os.path.join(temp_dir, "osm.csv")) as csv_file: - reader = csv.reader(csv_file) - rows = [row for row in reader] + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) + + with open(os.path.join(temp_dir, "osm.csv"), encoding="utf-8") as csv_file: + rows = list(csv.reader(csv_file)) expected_column_headers = [ "photo", @@ -2858,6 +2858,7 @@ def test_zipped_csv_export_with_osm_data(self): self.assertEqual(rows[1][6], "Patuatuli Road") self.assertEqual(rows[1][13], "kol") + # pylint: disable=invalid-name def test_zipped_sav_export_with_osm_data(self): """ Test that osm data is included in zipped sav export @@ -2884,17 +2885,15 @@ def test_zipped_sav_export_with_osm_data(self): ] export_builder = ExportBuilder() export_builder.set_survey(survey, xform) - temp_zip_file = NamedTemporaryFile(suffix=".zip") - export_builder.to_zipped_sav(temp_zip_file.name, osm_data) - temp_zip_file.seek(0) - temp_dir = tempfile.mkdtemp() - zip_file = zipfile.ZipFile(temp_zip_file.name, "r") - zip_file.extractall(temp_dir) - zip_file.close() - temp_zip_file.close() + with NamedTemporaryFile(suffix=".zip") as temp_zip_file: + export_builder.to_zipped_sav(temp_zip_file.name, osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + with zipfile.ZipFile(temp_zip_file.name, "r") as zip_file: + zip_file.extractall(temp_dir) with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: - rows = [r for r in reader] + rows = list(reader) expected_column_headers = [ x.encode("utf-8") for x in [ @@ -2969,7 +2968,7 @@ def test_show_choice_labels(self): self.assertEqual(expected_result, result) - def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 + def test_show_choice_labels_multi_language(self): # pylint: disable=invalid-name """ Test show_choice_labels=true for select one questions - multi language form. @@ -3003,7 +3002,7 @@ def test_show_choice_labels_multi_language(self): # pylint: disable=C0103 self.assertEqual(expected_result, result) - def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 + def test_show_choice_labels_select_multiple(self): # pylint: disable=invalid-name """ Test show_choice_labels=true for select multiple questions - split_select_multiples=true. @@ -3037,7 +3036,7 @@ def test_show_choice_labels_select_multiple(self): # pylint: disable=C0103 self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_1(self): """ Test show_choice_labels=true for select multiple questions @@ -3072,7 +3071,7 @@ def test_show_choice_labels_select_multiple_1(self): self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_2(self): """ Test show_choice_labels=true for select multiple questions @@ -3112,7 +3111,7 @@ def test_show_choice_labels_select_multiple_2(self): self.maxDiff = None self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language(self): """ Test show_choice_labels=true for select multiple questions - multi @@ -3148,7 +3147,7 @@ def test_show_choice_labels_select_multiple_language(self): self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_1(self): """ Test show_choice_labels=true, split_select_multiples=true, for select @@ -3187,7 +3186,7 @@ def test_show_choice_labels_select_multiple_language_1(self): self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_2(self): """ Test show_choice_labels=true, split_select_multiples=true, @@ -3228,7 +3227,7 @@ def test_show_choice_labels_select_multiple_language_2(self): self.assertEqual(expected_result, result) - # pylint: disable=C0103 + # pylint: disable=invalid-name def test_show_choice_labels_select_multiple_language_3(self): """ Test show_choice_labels=true, split_select_multiples=true, From 2996b71678fb0c4a93b578a2d1362bbfcc233c79 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 16:18:26 +0300 Subject: [PATCH 064/234] Cleanup --- onadata/apps/api/tests/viewsets/test_project_viewset.py | 8 +++----- onadata/libs/tests/utils/test_export_builder.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index d388c13b22..63beebf533 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -569,7 +569,7 @@ def test_view_xls_form(self): ("file_hash", None), ( "url", - "http://testserver/api/v1/metadata/{preview_url.pk}", + f"http://testserver/api/v1/metadata/{preview_url.pk}", ), ("date_created", preview_url.date_created), ] @@ -586,7 +586,7 @@ def test_view_xls_form(self): ("file_hash", None), ( "url", - "http://testserver/api/v1/metadata/{single_submit_url.pk}", + f"http://testserver/api/v1/metadata/{single_submit_url.pk}", ), ("date_created", single_submit_url.date_created), ] @@ -2575,9 +2575,7 @@ def test_project_caching(self): response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data["forms"]), 1) - self.assertEqual( - response.data["forms"][0]["name"], self.xform.title, response.data - ) + self.assertEqual(response.data["forms"][0]["name"], self.xform.title) self.assertEqual( response.data["forms"][0]["last_submission_time"], self.xform.time_of_last_submission(), diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 0ec03bd342..832278ad2e 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -2113,7 +2113,7 @@ def test_zipped_csv_export_remove_group_name(self): self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open( - os.path.join(temp_dir, "children.info.csv", encoding="utf-8") + os.path.join(temp_dir, "children.info.csv"), encoding="utf-8" ) as csv_file: reader = csv.reader(csv_file) expected_headers = [ From b6cdd80bec2a14198a3f583d2c30294d0fc54c56 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 16:36:49 +0300 Subject: [PATCH 065/234] Skip SPSS tests --- .../libs/tests/utils/test_export_builder.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 832278ad2e..41ea9e8e5b 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -13,12 +13,17 @@ from collections import OrderedDict from ctypes import ArgumentError from io import BytesIO +from unittest import skipIf from openpyxl import load_workbook from django.conf import settings from django.core.files.temp import NamedTemporaryFile from pyxform.builder import create_survey_from_xls -from savReaderWriter import SavHeaderReader, SavReader + +try: + from savReaderWriter import SavHeaderReader, SavReader +except ImportError: + SavHeaderReader = SavReader = None from onadata.apps.logger.import_tools import django_file from onadata.apps.main.tests.test_base import TestBase @@ -576,6 +581,7 @@ def test_zipped_csv_export_works_with_unicode(self): # check that red and blue are set to true # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_date_field(self): md = """ | survey | @@ -621,6 +627,7 @@ def test_zipped_sav_export_with_date_field(self): shutil.rmtree(temp_dir) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_dynamic_select_multiple(self): md = """ | survey | @@ -686,6 +693,7 @@ def test_zipped_sav_export_dynamic_select_multiple(self): shutil.rmtree(temp_dir) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_zero_padded_select_one_field(self): md = """ | survey | @@ -718,6 +726,7 @@ def test_zipped_sav_export_with_zero_padded_select_one_field(self): self.assertEqual(rows[1][4].decode("utf-8"), "2016-11-21 03:43:43") # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_numeric_select_one_field(self): md = """ | survey | @@ -769,6 +778,7 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_field_different_groups(self): """ Test SAV exports duplicate fields, same group - one field in repeat @@ -908,6 +918,7 @@ def test_split_select_multiples_choices_with_randomize_param(self): self.assertEqual(choices, expected_choices) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_numeric_select_multiple_field(self): md = """ | survey | | | | | @@ -982,6 +993,7 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): shutil.rmtree(temp_dir) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): md = """ | survey | | | | @@ -1020,6 +1032,7 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): shutil.rmtree(temp_dir) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_values_split_select_multiple(self): md = """ | survey | | | | @@ -1061,6 +1074,7 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): shutil.rmtree(temp_dir) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): md = """ | survey | | | | @@ -1100,6 +1114,7 @@ def test_zipped_sav_export_with_duplicate_name_in_choice_list(self): self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_external_choices(self): # pylint: disable=invalid-name """ Test that an SPSS export does not fail when it has choices from a file. @@ -1126,6 +1141,7 @@ def test_zipped_sav_export_external_choices(self): # pylint: disable=invalid-na self.assertTrue(os.path.exists(os.path.join(temp_dir, "exp.sav"))) # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_duplicate_column_name(self): """ Test that SAV exports with duplicate column names @@ -1940,6 +1956,7 @@ def test_convert_types(self): self.assertIsInstance(converted_val, datetime.date) self.assertEqual(converted_val, expected_val) + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -1993,6 +2010,7 @@ def _test_sav_file(section): section_name = section["name"].replace("/", "_") _test_sav_file(section_name) + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_to_sav_export_language(self): survey = self._create_childrens_survey("childrens_survey_sw.xlsx") export_builder = ExportBuilder() @@ -2345,6 +2363,7 @@ def test_zipped_csv_export_with_labels_only(self): # check that red and blue are set to true shutil.rmtree(temp_dir) + @skipIf(SavHeaderReader is None, "savReaderWriter is not supported now.") def test_to_sav_export_with_labels(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() @@ -2738,6 +2757,7 @@ def test_xls_export_has_submission_review_fields(self): self.assertEqual(xls_data[31], "2021-05-25T02:27:19") # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_has_submission_review_fields(self): """ Test that review comment, status and date fields are in csv exports @@ -2859,6 +2879,7 @@ def test_zipped_csv_export_with_osm_data(self): self.assertEqual(rows[1][13], "kol") # pylint: disable=invalid-name + @skipIf(SavReader is None, "savReaderWriter is not supported now.") def test_zipped_sav_export_with_osm_data(self): """ Test that osm data is included in zipped sav export From 3162a371a8e6917b2b4c3a6a64e5c480c2ed41dd Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 16:49:54 +0300 Subject: [PATCH 066/234] briefcase_viewset.py: cleanup --- .../apps/api/viewsets/briefcase_viewset.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index cb64b0f4b8..09d4a29e50 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.files import File from django.core.validators import ValidationError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.http import Http404 from django.utils.translation import ugettext as _ @@ -38,12 +38,15 @@ from onadata.libs.utils.viewer_tools import get_form +User = get_user_model() + + def _extract_uuid(text): if isinstance(text, six.string_types): form_id_parts = text.split("/") if form_id_parts.__len__() < 2: - raise ValidationError(_("Invalid formId %s." % text)) + raise ValidationError(_(f"Invalid formId {text}.")) text = form_id_parts[1] text = text[text.find("@key=") : -1].replace("@key=", "") @@ -65,7 +68,7 @@ def _parse_int(num): try: return num and int(num) except ValueError: - pass + return None class BriefcaseViewset( @@ -87,10 +90,11 @@ class BriefcaseViewset( serializer_class = XFormListSerializer template_name = "openrosa_response.xml" + # pylint: disable=unused-argument def get_object(self, queryset=None): - formId = self.request.GET.get("formId", "") - id_string = _extract_id_string(formId) - uuid = _extract_uuid(formId) + form_id = self.request.GET.get("formId", "") + id_string = _extract_id_string(form_id) + uuid = _extract_uuid(form_id) username = self.kwargs.get("username") obj = get_object_or_404( @@ -103,6 +107,7 @@ def get_object(self, queryset=None): return obj + # pylint: disable=too-many-branches def filter_queryset(self, queryset): username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: @@ -118,7 +123,7 @@ def filter_queryset(self, queryset): else: queryset = queryset.filter(user=profile.user) else: - queryset = super(BriefcaseViewset, self).filter_queryset(queryset) + queryset = super().filter_queryset(queryset) formId = self.request.GET.get("formId", "") @@ -175,8 +180,8 @@ def create(self, request, *args, **kwargs): response_status = status.HTTP_201_CREATED username = kwargs.get("username") form_user = ( - username and get_object_or_404(User, username=username) - ) or request.user + get_object_or_404(User, username=username) if username else request.user + ) if not request.user.has_perm("can_add_xform", form_user.profile): raise exceptions.PermissionDenied( @@ -190,12 +195,14 @@ def create(self, request, *args, **kwargs): if isinstance(xform_def, File): do_form_upload = PublishXForm(xform_def, form_user) - dd = publish_form(do_form_upload.publish_xform) + data_dictionary = publish_form(do_form_upload.publish_xform) - if isinstance(dd, XForm): - data["message"] = _("%s successfully published." % dd.id_string) + if isinstance(data_dictionary, XForm): + data["message"] = _( + f"{data_dictionary.id_string} successfully published." + ) else: - data["message"] = dd["text"] + data["message"] = data_dictionary["text"] response_status = status.HTTP_400_BAD_REQUEST else: data["message"] = _("Missing xml file.") @@ -227,9 +234,7 @@ def retrieve(self, request, *args, **kwargs): xml_obj = clean_and_parse_xml(self.object.xml) submission_xml_root_node = xml_obj.documentElement - submission_xml_root_node.setAttribute( - "instanceID", "uuid:%s" % self.object.uuid - ) + submission_xml_root_node.setAttribute("instanceID", f"uuid:{self.object.uuid}") submission_xml_root_node.setAttribute( "submissionDate", self.object.date_created.isoformat() ) From 578d11094f911a9f27f22a9da4f2c85fdef64825 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 17:04:39 +0300 Subject: [PATCH 067/234] renderers.py: cleanup --- onadata/libs/renderers/renderers.py | 61 +++++++++++++++-------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 077c189135..eb099775f4 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -92,7 +92,7 @@ def default(self, obj): # pylint: disable=method-hidden return JSONEncoder.default(self, obj) -class XLSRenderer(BaseRenderer): # pylint: disable=R0903 +class XLSRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ XLSRenderer - renders .xls spreadsheet documents with application/vnd.openxmlformats. @@ -117,7 +117,8 @@ class XLSXRenderer(XLSRenderer): # pylint: disable=too-few-public-methods format = "xlsx" -class CSVRenderer(BaseRenderer): # pylint: disable=abstract-method, R0903 +# pylint: disable=abstract-method, too-few-public-methods +class CSVRenderer(BaseRenderer): """ XLSRenderer - renders comma separated files (CSV) with text/csv. """ @@ -127,7 +128,7 @@ class CSVRenderer(BaseRenderer): # pylint: disable=abstract-method, R0903 charset = "utf-8" -class CSVZIPRenderer(BaseRenderer): # pylint: disable=R0903 +class CSVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ CSVZIPRenderer - renders a ZIP file that contains CSV files. """ @@ -139,7 +140,7 @@ class CSVZIPRenderer(BaseRenderer): # pylint: disable=R0903 def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): return data.encode("utf-8") - elif isinstance(data, dict): + if isinstance(data, dict): return json.dumps(data) return data @@ -156,7 +157,7 @@ class SAVZIPRenderer(BaseRenderer): # pylint: disable=too-few-public-methods def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): return data.encode("utf-8") - elif isinstance(data, dict): + if isinstance(data, dict): return json.dumps(data) return data @@ -187,7 +188,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return data -class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=R0903 +class GoogleSheetsRenderer(XLSRenderer): # pylint: disable=too-few-public-methods """ GoogleSheetsRenderer = Google Sheets excel exports. """ @@ -201,7 +202,8 @@ class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): matching format. """ - def filter_renderers(self, renderers, format): # pylint: disable=W0622 + # pylint: disable=redefined-builtin + def filter_renderers(self, renderers, format): """ If there is a '.json' style format suffix, filter the renderers so that we only negotiation against those that accept that format. @@ -214,7 +216,7 @@ def filter_renderers(self, renderers, format): # pylint: disable=W0622 return renderers -class MediaFileRenderer(BaseRenderer): # pylint: disable=R0903 +class MediaFileRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ MediaFileRenderer - render binary media files. """ @@ -230,7 +232,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return data -class XFormListRenderer(BaseRenderer): # pylint: disable=R0903 +class XFormListRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ Renderer which serializes to XML. """ @@ -248,7 +250,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): """ if data is None: return "" - elif isinstance(data, six.string_types): + if isinstance(data, six.string_types): return data stream = BytesIO() @@ -287,7 +289,8 @@ def _to_xml(self, xml, data): xml.characters(smart_text(data)) -class XFormManifestRenderer(XFormListRenderer): # pylint: disable=R0903 +# pylint: disable=too-few-public-methods +class XFormManifestRenderer(XFormListRenderer): """ XFormManifestRenderer - render XFormManifest XML. """ @@ -297,7 +300,8 @@ class XFormManifestRenderer(XFormListRenderer): # pylint: disable=R0903 xmlns = "http://openrosa.org/xforms/xformsManifest" -class TemplateXMLRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 +# pylint: disable=too-few-public-methods +class TemplateXMLRenderer(TemplateHTMLRenderer): """ TemplateXMLRenderer - Render XML template. """ @@ -312,9 +316,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): if response and response.exception: return XMLRenderer().render(data, accepted_media_type, renderer_context) - return super(TemplateXMLRenderer, self).render( - data, accepted_media_type, renderer_context - ) + return super().render(data, accepted_media_type, renderer_context) class InstanceXMLRenderer(XMLRenderer): @@ -331,6 +333,7 @@ def _get_current_buffer_data(self): self.stream.truncate(0) self.stream.seek(0) return ret + return None def stream_data(self, data, serializer): if data is None: @@ -441,7 +444,7 @@ def _to_xml(self, xml, data): xml.characters(force_str(data)) -class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=R0903 +class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=too-few-public-methods """ StaticXMLRenderer - render static XML document. """ @@ -450,7 +453,7 @@ class StaticXMLRenderer(StaticHTMLRenderer): # pylint: disable=R0903 media_type = "text/xml" -class GeoJsonRenderer(BaseRenderer): # pylint: disable=R0903 +class GeoJsonRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ GeoJsonRenderer - render .geojson data as json. """ @@ -463,7 +466,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return json.dumps(data) -class OSMRenderer(BaseRenderer): # pylint: disable=R0903 +class OSMRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ OSMRenderer - render .osm data as XML. """ @@ -490,7 +493,8 @@ def _list(list_or_item): return get_combined_osm(data) -class OSMExportRenderer(BaseRenderer): # pylint: disable=R0903, W0223 +# pylint: disable=too-few-public-methods,abstract-method +class OSMExportRenderer(BaseRenderer): """ OSMExportRenderer - render .osm data as XML. """ @@ -500,7 +504,8 @@ class OSMExportRenderer(BaseRenderer): # pylint: disable=R0903, W0223 charset = "utf-8" -class DebugToolbarRenderer(TemplateHTMLRenderer): # pylint: disable=R0903 +# pylint: disable=too-few-public-methods +class DebugToolbarRenderer(TemplateHTMLRenderer): """ DebugToolbarRenderer - render .debug as HTML. """ @@ -518,12 +523,10 @@ def render(self, data, accepted_media_type=None, renderer_context=None): ) } - return super(DebugToolbarRenderer, self).render( - data, accepted_media_type, renderer_context - ) + return super().render(data, accepted_media_type, renderer_context) -class ZipRenderer(BaseRenderer): # pylint: disable=R0903 +class ZipRenderer(BaseRenderer): # pylint: disable=too-few-public-methods """ ZipRenderer - render .zip files. """ @@ -535,7 +538,7 @@ class ZipRenderer(BaseRenderer): # pylint: disable=R0903 def render(self, data, accepted_media_type=None, renderer_context=None): if isinstance(data, six.text_type): return data.encode("utf-8") - elif isinstance(data, dict): + if isinstance(data, dict): return json.dumps(data) return data @@ -563,10 +566,8 @@ def render(self, data, accepted_media_type=None, renderer_context=None): results = data if request.method == "GET" and response.status_code == 200: if isinstance(data, dict): - results = [i for i in floip_rows_list(data)] + results = list(floip_rows_list(data)) else: - results = [i for i in floip_list(data)] + results = list(floip_list(data)) - return super(FLOIPRenderer, self).render( - results, accepted_media_type, renderer_context - ) + return super().render(results, accepted_media_type, renderer_context) From 8add763c29a5a94ec7d827ee65e6237075066d37 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 17:28:49 +0300 Subject: [PATCH 068/234] filters.py: cleanup --- onadata/libs/filters.py | 92 +++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 8be56809c6..1e00df7cbb 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -2,7 +2,7 @@ from uuid import UUID -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q @@ -20,6 +20,9 @@ from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms +User = get_user_model() + + class AnonDjangoObjectPermissionFilter(ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): """ @@ -45,19 +48,16 @@ def filter_queryset(self, request, queryset, view): else: xform_kwargs = {lookup_field: form_id} form = queryset.get(**xform_kwargs) - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object # Check if form is public and return it if form.shared: if lookup_field == "uuid": return queryset.filter(Q(uuid=form_id.hex) | Q(uuid=str(form_id))) - else: - return queryset.filter(Q(**xform_kwargs)) + return queryset.filter(Q(**xform_kwargs)) - return super(AnonDjangoObjectPermissionFilter, self).filter_queryset( - request, queryset, view - ) + return super().filter_queryset(request, queryset, view) # pylint: disable=too-few-public-methods @@ -74,16 +74,14 @@ def filter_queryset(self, request, queryset, view): self.perm_format = ( "%(app_label)s.report_%(model_name)s" # noqa pylint: disable=W0201 ) - return super(EnketoAnonDjangoObjectPermissionFilter, self).filter_queryset( - request, queryset, view - ) + return super().filter_queryset(request, queryset, view) class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): perm_format = "%(app_label)s.report_%(model_name)s" -class XFormListXFormPKFilter(object): +class XFormListXFormPKFilter: def filter_queryset(self, request, queryset, view): xform_pk = view.kwargs.get("xform_pk") if xform_pk: @@ -100,7 +98,8 @@ def filter_queryset(self, request, queryset, view): class FormIDFilter(django_filter_filters.FilterSet): - formID = django_filter_filters.CharFilter(field_name="id_string") + + formID = django_filter_filters.CharFilter(field_name="id_string") # noqa class Meta: model = XForm @@ -114,9 +113,7 @@ def filter_queryset(self, request, queryset, view): if view.action == "retrieve" and request.method == "GET": return queryset.model.objects.all() - filtered_queryset = super(OrganizationPermissionFilter, self).filter_queryset( - request, queryset, view - ) + filtered_queryset = super().filter_queryset(request, queryset, view) org_users = set( [group.team.organization for group in request.user.groups.all()] + [o.user for o in filtered_queryset] @@ -235,21 +232,19 @@ def filter_queryset(self, request, queryset, view): if project_id: int_or_parse_error( project_id, - "Invalid value for project_id. It must be a" " positive integer.", + "Invalid value for project_id. It must be a positive integer.", ) # check if project is public and return it try: project = queryset.get(id=project_id) - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object if project.shared: return queryset.filter(Q(id=project_id)) - return super(AnonUserProjectFilter, self).filter_queryset( - request, queryset, view - ) + return super().filter_queryset(request, queryset, view) class TagFilter(filters.BaseFilterBackend): @@ -264,7 +259,7 @@ def filter_queryset(self, request, queryset, view): return queryset -class XFormPermissionFilterMixin(object): +class XFormPermissionFilterMixin: def _xform_filter(self, request, view, keyword): """Use XForm permissions""" @@ -272,7 +267,7 @@ def _xform_filter(self, request, view, keyword): public_forms = XForm.objects.none() if xform: int_or_parse_error( - xform, "Invalid value for formid. It must be a" " positive integer." + xform, "Invalid value for formid. It must be a positive integer." ) self.xform = get_object_or_404(XForm, pk=xform) xform_qs = XForm.objects.filter(pk=self.xform.pk) @@ -284,27 +279,22 @@ def _xform_filter(self, request, view, keyword): if request.user.is_anonymous: xforms = xform_qs.filter(shared_data=True) else: - xforms = ( - super(XFormPermissionFilterMixin, self).filter_queryset( - request, xform_qs, view - ) - | public_forms - ) - return {"%s__in" % keyword: xforms} + xforms = super().filter_queryset(request, xform_qs, view) | public_forms + return {f"{keyword}__in": xforms} def _xform_filter_queryset(self, request, queryset, view, keyword): kwarg = self._xform_filter(request, view, keyword) return queryset.filter(**kwarg) -class ProjectPermissionFilterMixin(object): +class ProjectPermissionFilterMixin: def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") if project_id: int_or_parse_error( project_id, - "Invalid value for projectid. It must be a" " positive integer.", + "Invalid value for projectid. It must be a positive integer.", ) project = get_object_or_404(Project, pk=project_id) @@ -312,11 +302,9 @@ def _project_filter(self, request, view, keyword): else: project_qs = Project.objects.all() - projects = super(ProjectPermissionFilterMixin, self).filter_queryset( - request, project_qs, view - ) + projects = super().filter_queryset(request, project_qs, view) - return {"%s__in" % keyword: projects} + return {f"{keyword}__in": projects} def _project_filter_queryset(self, request, queryset, view, keyword): """Use Project Permissions""" @@ -325,7 +313,8 @@ def _project_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) -class InstancePermissionFilterMixin(object): +class InstancePermissionFilterMixin: + # pylint: disable=too-many-locals def _instance_filter(self, request, view, keyword): instance_kwarg = {} instance_content_type = ContentType.objects.get_for_model(Instance) @@ -356,21 +345,17 @@ def _instance_filter(self, request, view, keyword): project_qs = Project.objects.filter(pk=project.id) if parent and parent.project == project: - projects = super(InstancePermissionFilterMixin, self).filter_queryset( - request, project_qs, view - ) + projects = super().filter_queryset(request, project_qs, view) instances = [instance.id] if projects else [] - instance_kwarg["%s__in" % keyword] = instances + instance_kwarg[f"{keyword}__in"] = instances return instance_kwarg - else: - return {} + return {} - else: - return instance_kwarg + return instance_kwarg def _instance_filter_queryset(self, request, queryset, view, keyword): kwarg = self._instance_filter(request, view, keyword) @@ -414,12 +399,12 @@ def filter_queryset(self, request, queryset, view): if (xform_id and instance_kwarg) else [] ) - elif xform_id: + if xform_id: # return xform specific metadata return queryset.filter(Q(**xform_kwarg)) # return project specific metadata - elif project_id: + if project_id: return queryset.filter(Q(**project_kwarg)) # return all project,instance and xform metadata information @@ -517,8 +502,8 @@ def filter_queryset(self, request, queryset, view): return filtered_queryset - except ObjectDoesNotExist: - raise Http404 + except ObjectDoesNotExist as non_existent_object: + raise Http404 from non_existent_object return queryset @@ -530,7 +515,7 @@ def filter_queryset(self, request, queryset, view): # Return widgets from xform user has perms to return self._xform_filter_queryset(request, queryset, view, "object_id") - return super(WidgetFilter, self).filter_queryset(request, queryset, view) + return super().filter_queryset(request, queryset, view) class UserProfileFilter(filters.BaseFilterBackend): @@ -540,7 +525,7 @@ def filter_queryset(self, request, queryset, view): if users: users = users.split(",") return queryset.filter(user__username__in=users) - elif not request.user.is_anonymous: + if not request.user.is_anonymous: return queryset.filter(user__username=request.user.username) return queryset.none() @@ -607,7 +592,8 @@ def filter_queryset(self, request, queryset, view): return all_qs | submitter_qs -class PublicDatasetsFilter(object): +class PublicDatasetsFilter: + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): if request and request.user.is_anonymous: return queryset.filter(shared=True) From 68926e7b978ee1cd015d168812a4b3a5811ca4ca Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 19:59:35 +0300 Subject: [PATCH 069/234] batch: cleanup --- .github/ISSUE_TEMPLATE.md | 15 -- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- ...{feature_request.md => feature-request.md} | 0 .github/PULL_REQUEST_TEMPLATE.md | 7 +- CONTRIBUTING.MD | 30 ++-- docker/docker-entrypoint.sh | 6 +- docker/postgis/Dockerfile | 9 +- onadata/apps/api/admin.py | 23 ++- .../commands/fix_readonly_role_perms.py | 70 +++++---- .../api/tests/viewsets/test_note_viewset.py | 135 +++++++++--------- onadata/apps/logger/models/instance.py | 95 ++++++------ .../logger/tests/test_briefcase_client.py | 10 +- onadata/libs/utils/decorators.py | 7 +- 13 files changed, 210 insertions(+), 199 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md rename .github/ISSUE_TEMPLATE/{feature_request.md => feature-request.md} (100%) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index f64bfadd8a..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,15 +0,0 @@ -### Environmental Information - -- Operating System: -- Onadata version: -- Python version: - -### Problem description - -### Expected behavior - -### Steps to reproduce the behavior - -### Additional Information - -_Logs, [related issues](github.com/onaio/onadata/issues), weird / out of place occurrences, local settings, possible approach to solving this..._ diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 4b65576226..30b90eddb8 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -9,7 +9,7 @@ assignees: '' ### Environmental Information -- Onadata version: + - Onadata version: ### Problem description diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature-request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature-request.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8d4b447778..b6c3be67e5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,9 +4,10 @@ ### Side effects of implementing this change -### Before submitting this PR for review, please make sure you have: -- [ ] Included tests -- [ ] Updated documentation +**Before submitting this PR for review, please make sure you have:** + + - [ ] Included tests + - [ ] Updated documentation Closes # diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index b3746493fc..c7b38f4b24 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -10,37 +10,37 @@ The following is a set of guidelines for contributing to Ona Data. Following the In case you have encountered any issue within the project, please make sure: -- You are using the [latest release](http://github.com/onaio/onadata/releases). -- You have setup the project according to the [Installation Documentation](https://api.ona.io/static/docs/install.html). + - You are using the [latest release](http://github.com/onaio/onadata/releases). + - You have setup the project according to the [Installation Documentation](https://api.ona.io/static/docs/install.html). After confirming the above, make sure the issue has not been reported on our [issues page](https://github.com/onaio/onadata/issues). If it hasn't been reported, [open a ticket](https://github.com/onaio/onadata/issues/new) containing: -- Information about your system environment (Operating System, local settings, etc.). -- What you expected to happen, and what actually happened. -- Out of place / weird logs and any other interesting information -- All steps to reproduce the issue. + - Information about your system environment (Operating System, local settings, etc.). + - What you expected to happen, and what actually happened. + - Out of place / weird logs and any other interesting information + - All steps to reproduce the issue. ### 2. Suggest Enhancements or New Features ⚡ Feature and enhancement requests are always welcome! We ask you ensure the following details are provided while [opening a ticket](https://github.com/onaio/onadata/issues/new), in order to start a constructive discussion: -- Describe the feature/enhancement in detail. -- Explain why the feature/enhancement is needed. -- Describe how the feature/enhancement should work -- List any advantages & disadvantages of implementing the feature/enhancement + - Describe the feature/enhancement in detail. + - Explain why the feature/enhancement is needed. + - Describe how the feature/enhancement should work + - List any advantages & disadvantages of implementing the feature/enhancement ### 3. Code contributions / Pull requests 💻 Pull requests are wholeheartedly welcome!❤️ If you are unsure about how to make your first pull request, here are some helpful resources: -- [Creating a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) -- [How to create effective pull requests](https://dev.to/mpermar/how-to-create-effective-pull-requests-2m8e) + - [Creating a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) + - [How to create effective pull requests](https://dev.to/mpermar/how-to-create-effective-pull-requests-2m8e) In order to make it easier for us to facilitate the smooth merging of your pull request, please make sure the following standards are met within your pull request. -- Ensure your git commits are signed. _Read about signing commits [here](https://help.github.com/en/github/authenticating-to-github/signing-commits)_ -- Code & commits follow our [styleguides](#Styleguides). -- Implement / Update tests that need to be updated and ensure that they pass. _Running all the tests within this project may be tough and time consuming. But not to worry! On submission of the pull request, [Travis CI](https://travis-ci.org/) will run all the tests across our modules_. + - Ensure your git commits are signed. _Read about signing commits [here](https://help.github.com/en/github/authenticating-to-github/signing-commits)_ + - Code & commits follow our [styleguides](#Styleguides). + - Implement / Update tests that need to be updated and ensure that they pass. _Running all the tests within this project may be tough and time consuming. But not to worry! On submission of the pull request, [Travis CI](https://travis-ci.org/) will run all the tests across our modules_. With the above points in mind feel free to comment on one of our [beginner-friendly issues](https://github.com/onaio/onadata/issues?q=is%3Aissue+is%3Aopen+label%3A%22Good+First+Issue%22) expressing your intent to work on it. diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 12b2ac667c..e38e40f4dd 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ ${INITDB} ]; then +if [ "${INITDB}" ]; then RUN_DB_INIT_SCRIPT=$INITDB else RUN_DB_INIT_SCRIPT=true @@ -13,8 +13,8 @@ if $RUN_DB_INIT_SCRIPT; then psql -h db -U postgres onadata -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" fi -virtualenv -p `which $SELECTED_PYTHON` /srv/onadata/.virtualenv/${SELECTED_PYTHON} -. /srv/onadata/.virtualenv/${SELECTED_PYTHON}/bin/activate +virtualenv -p "$(which ${SELECTED_PYTHON})" /srv/onadata/.virtualenv/"${SELECTED_PYTHON}" +. /srv/onadata/.virtualenv/"${SELECTED_PYTHON}"/bin/activate cd /srv/onadata pip install --upgrade pip diff --git a/docker/postgis/Dockerfile b/docker/postgis/Dockerfile index b914a1372c..f2d491e497 100644 --- a/docker/postgis/Dockerfile +++ b/docker/postgis/Dockerfile @@ -1,12 +1,11 @@ FROM postgres:9.6 -MAINTAINER Ukang'a Dickson - - RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive \ - apt-get install -y postgresql-9.6-postgis-2.3 \ - postgresql-9.6-postgis-script postgis \ + apt-get install --no-install-recommends -y \ + postgresql-9.6-postgis-2.3=2.3.1+dfsg-2+deb9u2 \ + postgresql-9.6-postgis-2.3-scripts=2.3.1+dfsg-2+deb9u2 \ + postgis=2.3.1+dfsg-2+deb9u2 \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/onadata/apps/api/admin.py b/onadata/apps/api/admin.py index d4a9ea6dbd..a10f339dfd 100644 --- a/onadata/apps/api/admin.py +++ b/onadata/apps/api/admin.py @@ -1,39 +1,38 @@ +# -*- coding: utf-8 -*- +"""API Django admin amendments.""" from django.contrib import admin from onadata.apps.api.models import Team, OrganizationProfile, TempToken class TeamAdmin(admin.ModelAdmin): - def get_queryset(self, request): - qs = super(TeamAdmin, self).get_queryset(request) + queryset = super(TeamAdmin, self).get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(user=request.user) admin.site.register(Team, TeamAdmin) class OrganizationProfileAdmin(admin.ModelAdmin): - def get_queryset(self, request): - qs = super(OrganizationProfileAdmin, self).get_queryset(request) + queryset = super(OrganizationProfileAdmin, self).get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(user=request.user) admin.site.register(OrganizationProfile, OrganizationProfileAdmin) class TempTokenProfileAdmin(admin.ModelAdmin): - def get_queryset(self, request): - qs = super(TempTokenProfileAdmin, self).get_queryset(request) + queryset = super(TempTokenProfileAdmin, self).get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(user=request.user) admin.site.register(TempToken, TempTokenProfileAdmin) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 9a76d0ce74..268afe4fb1 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -1,30 +1,41 @@ from guardian.shortcuts import get_perms from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from django.conf import settings from onadata.apps.api.models import Team -from onadata.libs.permissions import ReadOnlyRole, DataEntryRole,\ - EditorRole, ManagerRole, OwnerRole, ReadOnlyRoleNoDownload,\ - DataEntryOnlyRole, DataEntryMinorRole, EditorMinorRole +from onadata.libs.permissions import ( + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRoleNoDownload, + DataEntryOnlyRole, + DataEntryMinorRole, + EditorMinorRole, +) from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() + + class Command(BaseCommand): - args = '' - help = _(u"Reassign permission to the model when permissions are changed") + args = "" + help = _("Reassign permission to the model when permissions are changed") def handle(self, *args, **options): - self.stdout.write("Re-assigining started", ending='\n') + self.stdout.write("Re-assigining started", ending="\n") if not args: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") if len(args) < 3: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") app = args[0] model = args[1] @@ -47,8 +58,9 @@ def handle(self, *args, **options): for team in queryset_iterator(teams): self.reassign_perms(team, app, model, new_perms) - self.stdout.write("Re-assigining finished", ending='\n') + self.stdout.write("Re-assigining finished", ending="\n") + # pylint: disable=unused-argument def reassign_perms(self, user, app, model, new_perm): """ Gets all the permissions the user has on objects and assigns the new @@ -65,33 +77,39 @@ def reassign_perms(self, user, app, model, new_perm): if isinstance(user, Team): if model == "project": objects = user.projectgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: objects = user.xformgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: - if model == 'project': + if model == "project": objects = user.projectuserobjectpermission_set.all() else: objects = user.xformuserobjectpermission_set.all() for perm_obj in objects: obj = perm_obj.content_object - ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryOnlyRole, - DataEntryMinorRole, - DataEntryRole, - EditorMinorRole, - EditorRole, - ManagerRole, - OwnerRole] + ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ] # For each role reassign the perms for role_class in reversed(ROLES): # want to only process for readonly perms - if role_class.user_has_role(user, obj) or role_class \ - not in [ReadOnlyRoleNoDownload, ReadOnlyRole]: + if role_class.user_has_role(user, obj) or role_class not in [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + ]: continue if self.check_role(role_class, user, obj, new_perm): @@ -99,7 +117,7 @@ def reassign_perms(self, user, app, model, new_perm): role_class.add(user, obj) break - def check_role(self, role_class, user, obj, new_perm=[]): + def check_role(self, role_class, user, obj, new_perm=None): """ Test if the user has the role for the object provided :param role_class: @@ -108,6 +126,7 @@ def check_role(self, role_class, user, obj, new_perm=[]): :param new_perm: :return: """ + new_perm = new_perm if new_perm is None else [] # remove the new permission because the old model doesnt have it perm_list = role_class.class_to_permissions[type(obj)] old_perm_set = set(perm_list) @@ -120,3 +139,4 @@ def check_role(self, role_class, user, obj, new_perm=[]): return set(get_perms(user, obj)) == diff_set return user.has_perms(list(diff_set), obj) + return False diff --git a/onadata/apps/api/tests/viewsets/test_note_viewset.py b/onadata/apps/api/tests/viewsets/test_note_viewset.py index f73a227543..491724080d 100644 --- a/onadata/apps/api/tests/viewsets/test_note_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_note_viewset.py @@ -17,39 +17,38 @@ class TestNoteViewSet(TestBase): """ Test NoteViewSet """ + def setUp(self): super(TestNoteViewSet, self).setUp() self._create_user_and_login() self._publish_transportation_form() self._make_submissions() - self.view = NoteViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'delete': 'destroy' - }) + self.view = NoteViewSet.as_view( + {"get": "list", "post": "create", "delete": "destroy"} + ) self.factory = RequestFactory() - self.extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} @property def _first_xform_instance(self): - return self.xform.instances.all().order_by('pk')[0] + return self.xform.instances.all().order_by("pk")[0] def _add_notes_to_data_point(self): # add a note to a specific data point - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note, **self.extra) + note["instance"] = dataid + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 201) - self.pk = response.data['id'] - note['id'] = self.pk + self.pk = response.data["id"] + note["id"] = self.pk self.note = note def test_note_list(self): self._add_notes_to_data_point() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertTrue(len(response.data) > 0) @@ -57,29 +56,29 @@ def test_note_list(self): def test_note_get(self): self._add_notes_to_data_point() - view = NoteViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = NoteViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['owner'], self.user.username) + self.assertEqual(response.data["owner"], self.user.username) self.assertDictContainsSubset(self.note, response.data) def test_get_note_for_specific_instance(self): self._add_notes_to_data_point() - view = NoteViewSet.as_view({'get': 'retrieve'}) + view = NoteViewSet.as_view({"get": "retrieve"}) instance = self.xform.instances.first() query_params = {"instance": instance.id} - request = self.factory.get('/', data=query_params, **self.extra) + request = self.factory.get("/", data=query_params, **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) self.assertDictContainsSubset(self.note, response.data) second_instance = self.xform.instances.last() query_params = {"instance": second_instance.id} - request = self.factory.get('/', data=query_params, **self.extra) + request = self.factory.get("/", data=query_params, **self.extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) @@ -87,18 +86,18 @@ def test_get_note_for_specific_instance(self): def test_add_notes_to_data_point(self): self._add_notes_to_data_point() - self.assertEquals(len(self._first_xform_instance.json["_notes"]), 1) + self.assertEqual(len(self._first_xform_instance.json["_notes"]), 1) def test_other_user_notes_access(self): - self._create_user_and_login('lilly', '1234') - extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - note = {'note': u"Road Warrior"} + self._create_user_and_login("lilly", "1234") + extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + note = {"note": "Road Warrior"} dataid = self.xform.instances.first().pk - note['instance'] = dataid + note["instance"] = dataid # Other user 'lilly' should not be able to create notes # to xform instance owned by 'bob' - request = self.factory.post('/', data=note) + request = self.factory.post("/", data=note) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 401) @@ -107,44 +106,44 @@ def test_other_user_notes_access(self): self._add_notes_to_data_point() # access to /notes endpoint,should be empty list - request = self.factory.get('/', **extra) + request = self.factory.get("/", **extra) response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) # Other user 'lilly' sees an empty list when accessing bob's notes - view = NoteViewSet.as_view({'get': 'retrieve'}) + view = NoteViewSet.as_view({"get": "retrieve"}) query_params = {"instance": dataid} - request = self.factory.get('/', data=query_params, **extra) + request = self.factory.get("/", data=query_params, **extra) response = view(request, pk=self.pk) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_collaborator_with_readonly_permission_can_add_comment(self): - self._create_user_and_login('lilly', '1234') - extra = {'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self._create_user_and_login("lilly", "1234") + extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} # save some notes self._add_notes_to_data_point() # post note to submission as lilly without permissions - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note) + note["instance"] = dataid + request = self.factory.post("/", data=note) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 401) # post note to submission with permissions to form - assign_perm('view_xform', self.user, self._first_xform_instance.xform) + assign_perm("view_xform", self.user, self._first_xform_instance.xform) - note = {'note': u"Road Warrior"} + note = {"note": "Road Warrior"} dataid = self._first_xform_instance.pk - note['instance'] = dataid - request = self.factory.post('/', data=note, **extra) + note["instance"] = dataid + request = self.factory.post("/", data=note, **extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) @@ -152,48 +151,40 @@ def test_collaborator_with_readonly_permission_can_add_comment(self): def test_delete_note(self): self._add_notes_to_data_point() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.pk) self.assertEqual(response.status_code, 204) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 200) - self.assertEquals(response.data, []) + self.assertEqual(response.data, []) def test_question_level_notes(self): field = "transport" dataid = self.xform.instances.all()[0].pk - note = { - 'note': "Road Warrior", - 'instance': dataid, - 'instance_field': field - } - request = self.factory.post('/', data=note, **self.extra) + note = {"note": "Road Warrior", "instance": dataid, "instance_field": field} + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 201) instance = self.xform.instances.all()[0] - self.assertEquals(len(instance.json["_notes"]), 1) + self.assertEqual(len(instance.json["_notes"]), 1, instance.json) note = instance.json["_notes"][0] - self.assertEquals(note['instance_field'], field) + self.assertEqual(note["instance_field"], field) def test_only_add_question_notes_to_existing_fields(self): field = "bla" dataid = self.xform.instances.all()[0].pk - note = { - 'note': "Road Warrior", - 'instance': dataid, - 'instance_field': field - } - request = self.factory.post('/', data=note, **self.extra) + note = {"note": "Road Warrior", "instance": dataid, "instance_field": field} + request = self.factory.post("/", data=note, **self.extra) self.assertTrue(self.xform.instances.count()) response = self.view(request) self.assertEqual(response.status_code, 400) instance = self.xform.instances.all()[0] - self.assertEquals(len(instance.json["_notes"]), 0) + self.assertEqual(len(instance.json["_notes"]), 0) def test_csv_export_form_w_notes(self): """ @@ -208,26 +199,34 @@ def test_csv_export_form_w_notes(self): instance.save() instance.parsed_instance.save() - view = XFormViewSet.as_view({'get': 'retrieve'}) + view = XFormViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertTrue(response.status_code, 200) - test_file_path = os.path.join(settings.PROJECT_ROOT, 'apps', 'viewer', - 'tests', 'fixtures', - 'transportation_w_notes.csv') + test_file_path = os.path.join( + settings.PROJECT_ROOT, + "apps", + "viewer", + "tests", + "fixtures", + "transportation_w_notes.csv", + ) self._test_csv_response(response, test_file_path) def test_attribute_error_bug(self): """NoteSerializer: Should not raise AttributeError exeption""" - note = Note(note='Hello', instance=self._first_xform_instance) + note = Note(note="Hello", instance=self._first_xform_instance) note.save() data = NoteSerializer(note).data - self.assertDictContainsSubset({ - 'created_by': None, - 'note': u'Hello', - 'instance': note.instance_id, - 'owner': None - }, data) + self.assertDictContainsSubset( + { + "created_by": None, + "note": "Hello", + "instance": note.instance_id, + "owner": None, + }, + data, + ) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index e73fa09d5d..ce42961708 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -8,7 +8,7 @@ from deprecated import deprecated from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point from django.core.cache import cache @@ -77,32 +77,34 @@ ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False ) +User = get_user_model() def get_attachment_url(attachment, suffix=None): kwargs = {"pk": attachment.pk} - url = "{}?filename={}".format( - reverse("files-detail", kwargs=kwargs), attachment.media_file.name + url = ( + f"{reverse('files-detail', kwargs=kwargs)}" + f"?filename={attachment.media_file.name}" ) if suffix: - url += "&suffix={}".format(suffix) + url += f"&suffix={suffix}" return url def _get_attachments_from_instance(instance): attachments = [] - for a in instance.attachments.filter(deleted_at__isnull=True): - attachment = dict() - attachment["download_url"] = get_attachment_url(a) - attachment["small_download_url"] = get_attachment_url(a, "small") - attachment["medium_download_url"] = get_attachment_url(a, "medium") - attachment["mimetype"] = a.mimetype - attachment["filename"] = a.media_file.name - attachment["name"] = a.name - attachment["instance"] = a.instance.pk + for item in instance.attachments.filter(deleted_at__isnull=True): + attachment = {} + attachment["download_url"] = get_attachment_url(item) + attachment["small_download_url"] = get_attachment_url(item, "small") + attachment["medium_download_url"] = get_attachment_url(item, "medium") + attachment["mimetype"] = item.mimetype + attachment["filename"] = item.media_file.name + attachment["name"] = item.name + attachment["instance"] = item.instance.pk attachment["xform"] = instance.xform.id - attachment["id"] = a.id + attachment["id"] = item.id attachments.append(attachment) return attachments @@ -156,11 +158,7 @@ def get_id_string_from_xml_str(xml_str): elems = root_node.getElementsByTagName("data") for data in elems: - for child in data.childNodes: - id_string = data.childNodes[0].getAttribute("id") - - if len(id_string) > 0: - break + id_string = data.childNodes[0].getAttribute("id") if len(id_string) > 0: break @@ -206,6 +204,8 @@ def _update_submission_count_for_today( @transaction.atomic() def update_xform_submission_count(instance_id, created): if created: + + # pylint: disable=import-outside-toplevel from multidb.pinning import use_master with use_master: @@ -240,15 +240,16 @@ def update_xform_submission_count(instance_id, created): # Track submissions made today _update_submission_count_for_today(instance.xform_id) - safe_delete("{}{}".format(XFORM_DATA_VERSIONS, instance.xform_id)) - safe_delete("{}{}".format(DATAVIEW_COUNT, instance.xform_id)) - safe_delete("{}{}".format(XFORM_COUNT, instance.xform_id)) + safe_delete(f"{XFORM_DATA_VERSIONS}{instance.xform_id}") + safe_delete(f"{DATAVIEW_COUNT}{instance.xform_id}") + safe_delete(f"{XFORM_COUNT}{instance.xform_id}") # Clear project cache from onadata.apps.logger.models.xform import clear_project_cache clear_project_cache(instance.xform.project_id) +# pylint: disable=unused-argument,invalid-name def update_xform_submission_count_delete(sender, instance, **kwargs): try: xform = XForm.objects.select_for_update().get(pk=instance.xform.pk) @@ -256,8 +257,8 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): pass else: xform.num_of_submissions -= 1 - if xform.num_of_submissions < 0: - xform.num_of_submissions = 0 + + xform.num_of_submissions = max(xform.num_of_submissions, 0) xform.save(update_fields=["num_of_submissions"]) profile_qs = User.profile.get_queryset() try: @@ -266,8 +267,7 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): pass else: profile.num_of_submissions -= 1 - if profile.num_of_submissions < 0: - profile.num_of_submissions = 0 + profile.num_of_submissions = max(profile.num_of_submissions, 0) profile.save() # Track submissions made today @@ -275,13 +275,13 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): xform.id, incr=False, date_created=instance.date_created ) - for a in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: - safe_delete("{}{}".format(a, xform.project.pk)) + for cache_prefix in [PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE]: + safe_delete(f"{cache_prefix}{xform.project.pk}") - safe_delete("{}{}".format(IS_ORG, xform.pk)) - safe_delete("{}{}".format(XFORM_DATA_VERSIONS, xform.pk)) - safe_delete("{}{}".format(DATAVIEW_COUNT, xform.pk)) - safe_delete("{}{}".format(XFORM_COUNT, xform.pk)) + safe_delete(f"{IS_ORG}{xform.pk}") + safe_delete(f"{XFORM_DATA_VERSIONS}{xform.pk}") + safe_delete(f"{DATAVIEW_COUNT}{xform.pk}") + safe_delete(f"{XFORM_COUNT}{xform.pk}") if xform.instances.exclude(geom=None).count() < 1: xform.instances_with_geopoints = False @@ -301,6 +301,7 @@ def save_full_json(instance_id, created): instance.save(update_fields=["json"]) +# pylint: disable=unused-argument @app.task def update_project_date_modified(instance_id, created): # update the date modified field of the project which will change @@ -324,15 +325,16 @@ def convert_to_serializable_date(date): return date -class InstanceBaseClass(object): +class InstanceBaseClass: """Interface of functions for Instance and InstanceHistory model""" @property def point(self): - gc = self.geom + geom_collection = self.geom - if gc and len(gc): - return gc[0] + if geom_collection and len(geom_collection): + return geom_collection[0] + return None def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: @@ -372,7 +374,7 @@ def _set_geom(self): except ValueError: return - if not xform.instances_with_geopoints and len(points): + if not xform.instances_with_geopoints and points: xform.instances_with_geopoints = True xform.save() @@ -422,10 +424,11 @@ def get_full_dict(self, load_existing=True): if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() - # pylint: disable=attribute-defined-outside-init + # pylint: disable=attribute-defined-outside-init,access-member-before-definition if not self.date_created: self.date_created = submission_time() + # pylint: disable=access-member-before-definition if not self.date_modified: self.date_modified = self.date_created @@ -442,9 +445,10 @@ def get_full_dict(self, load_existing=True): edited = self.last_edited is not None doc[EDITED] = edited - edited and doc.update( - {LAST_EDITED: convert_to_serializable_date(self.last_edited)} - ) + if edited: + doc.update( + {LAST_EDITED: convert_to_serializable_date(self.last_edited)} + ) return doc def _set_parser(self): @@ -453,12 +457,12 @@ def _set_parser(self): self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): - self.survey_type, created = SurveyType.objects.get_or_create( + self.survey_type, _created = SurveyType.objects.get_or_create( slug=self.get_root_node_name() ) def _set_uuid(self): - # pylint: disable=no-member, attribute-defined-outside-init + # pylint: disable=no-member,attribute-defined-outside-init,access-member-before-definition if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) @@ -470,6 +474,7 @@ def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) + # pylint: disable=unused-argument def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() @@ -675,7 +680,7 @@ def save(self, *args, **kwargs): # pylint: disable=no-member self.version = self.json.get(VERSION, self.xform.version) - super(Instance, self).save(*args, **kwargs) + super().save(*args, **kwargs) # pylint: disable=no-member def set_deleted(self, deleted_at=timezone.now(), user=None): @@ -698,6 +703,7 @@ def soft_delete_attachments(self, user=None): queryset.update(**kwargs) +# pylint: disable=unused-argument def post_save_submission(sender, instance=None, created=False, **kwargs): if instance.deleted_at is not None: _update_submission_count_for_today( @@ -805,6 +811,7 @@ def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform_instance.xform) + # pylint: disable=unused-argument @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): return None diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 8cb0596f70..7f5e0e8958 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -66,9 +66,10 @@ def instances_xml(url, request, **kwargs): response = requests.Response() client = DigestClient() client.set_authorization("bob", "bob", "Digest") - res = client.get("%s?%s" % (url.path, url.query)) + res = client.get(f"{url.path}?{url.query}") if res.status_code == 302: res = client.get(res["Location"]) + assert res.status_code == 200, res.content response.encoding = res.get("content-type") response._content = get_streaming_content(res) else: @@ -108,16 +109,13 @@ def test_download_xform_xml(self): "deno", "briefcase", "forms", self.xform.id_string ) self.assertTrue(storage.exists(forms_folder_path)) - forms_path = os.path.join(forms_folder_path, "%s.xml" % self.xform.id_string) + forms_path = os.path.join(forms_folder_path, "{self.xform.id_string}.xml") self.assertTrue(storage.exists(forms_path)) form_media_path = os.path.join(forms_folder_path, "form-media") self.assertTrue(storage.exists(form_media_path)) media_path = os.path.join(form_media_path, "screenshot.png") self.assertTrue(storage.exists(media_path)) - """ - Download instance xml - """ with HTTMock(instances_xml): self.bc.download_instances(self.xform.id_string) instance_folder_path = os.path.join( @@ -131,7 +129,7 @@ def test_download_xform_xml(self): self.assertTrue(storage.exists(instance_path)) media_file = "1335783522563.jpg" media_path = os.path.join( - instance_folder_path, "uuid%s" % instance.uuid, media_file + instance_folder_path, "uuid{instance.uuid}", media_file ) self.assertTrue(storage.exists(media_path)) diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 55a0f6cc46..43d3518fc0 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -1,12 +1,16 @@ +# -*- coding: utf-8 -*- +"""decorators module""" from functools import wraps from six.moves.urllib.parse import urlparse from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.auth.views import redirect_to_login from django.conf import settings from django.http import HttpResponseRedirect def check_obj(f): + # pylint: disable=inconsistent-return-statements @wraps(f) def with_check_obj(*args, **kwargs): if args[0]: @@ -23,7 +27,7 @@ def _wrapped_view(request, *args, **kwargs): if request.user.username == kwargs["username"]: return view_func(request, *args, **kwargs) protocol = "https" if request.is_secure() else "http" - return HttpResponseRedirect("%s://%s" % (protocol, request.get_host())) + return HttpResponseRedirect(f"{protocol}://{request.get_host()}") path = request.build_absolute_uri() login_url = request.build_absolute_uri(settings.LOGIN_URL) # If the login url is the same scheme and net location then just @@ -34,7 +38,6 @@ def _wrapped_view(request, *args, **kwargs): not login_netloc or login_netloc == current_netloc ): path = request.get_full_path() - from django.contrib.auth.views import redirect_to_login return redirect_to_login(path, None, REDIRECT_FIELD_NAME) From 6e11628deece70f7299a203318b2bd7c0fb66641 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 20:49:57 +0300 Subject: [PATCH 070/234] batch: cleanup --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- docker/docker-entrypoint.sh | 2 +- onadata/apps/api/tests/viewsets/test_note_viewset.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 30b90eddb8..4b65576226 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -9,7 +9,7 @@ assignees: '' ### Environmental Information - - Onadata version: +- Onadata version: ### Problem description diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index e38e40f4dd..367a2b1cb5 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,7 +13,7 @@ if $RUN_DB_INIT_SCRIPT; then psql -h db -U postgres onadata -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" fi -virtualenv -p "$(which ${SELECTED_PYTHON})" /srv/onadata/.virtualenv/"${SELECTED_PYTHON}" +virtualenv -p "$(which ${SELECTED_PYTHON})" "/srv/onadata/.virtualenv/${SELECTED_PYTHON}" . /srv/onadata/.virtualenv/"${SELECTED_PYTHON}"/bin/activate cd /srv/onadata diff --git a/onadata/apps/api/tests/viewsets/test_note_viewset.py b/onadata/apps/api/tests/viewsets/test_note_viewset.py index 491724080d..9b6cba82d1 100644 --- a/onadata/apps/api/tests/viewsets/test_note_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_note_viewset.py @@ -168,7 +168,7 @@ def test_question_level_notes(self): response = self.view(request) self.assertEqual(response.status_code, 201) - instance = self.xform.instances.all()[0] + instance = self.xform.instances.get(pk=dataid) self.assertEqual(len(instance.json["_notes"]), 1, instance.json) note = instance.json["_notes"][0] From 6ae9c95140dccfe948b979f8915887b2d618e308 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 21:05:22 +0300 Subject: [PATCH 071/234] Indicate md5 use in gravatar is not used for security' --- onadata/libs/utils/gravatar.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index bae078432d..5e48f7329d 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,3 +1,5 @@ +# -*- codingL: utf-8 -*- +"""Gravatar utils module""" import hashlib from six.moves.urllib.parse import urlencode from six.moves.urllib.request import urlopen @@ -8,10 +10,14 @@ def email_md5(user): - return hashlib.md5(user.email.lower().encode("utf-8")).hexdigest() + """Returns the hash of an email for the user""" + return hashlib.new( + "md5", user.email.lower().encode("utf-8"), usedforsecurity=False + ).hexdigest() def get_gravatar_img_link(user): + """Returns the Gravatar image URL""" return ( GRAVATAR_ENDPOINT + email_md5(user) @@ -21,5 +27,6 @@ def get_gravatar_img_link(user): def gravatar_exists(user): + """Checks if the Gravatar URL exists""" url = GRAVATAR_ENDPOINT + email_md5(user) + "?" + "d=404" return urlopen(url).getcode() != 404 From 87130f7eaec232d5afbc90425442c8586d04a873 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 21:13:01 +0300 Subject: [PATCH 072/234] Use requests library since urllib.urlopen is blacklisted --- onadata/libs/utils/gravatar.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index 5e48f7329d..a89ec236f3 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,8 +1,9 @@ # -*- codingL: utf-8 -*- """Gravatar utils module""" import hashlib +import requests from six.moves.urllib.parse import urlencode -from six.moves.urllib.request import urlopen + DEFAULT_GRAVATAR = "https://ona.io/static/images/default_avatar.png" GRAVATAR_ENDPOINT = "https://secure.gravatar.com/avatar/" @@ -29,4 +30,4 @@ def get_gravatar_img_link(user): def gravatar_exists(user): """Checks if the Gravatar URL exists""" url = GRAVATAR_ENDPOINT + email_md5(user) + "?" + "d=404" - return urlopen(url).getcode() != 404 + return requests.get(url).status_code != 404 From 7b228053d4eaa824ca07f1c1bfef77518990479f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 21:21:05 +0300 Subject: [PATCH 073/234] cleanup --- onadata/libs/models/signals.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 3193b1a85c..a52ea40626 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,13 +1,17 @@ +# -*- codingL: utf-8 -*- +"""onadata.libs.models.signals module""" import django.dispatch from onadata.apps.logger.models import XForm -xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) -xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) +XFORM_TAGS_ADD = django.dispatch.Signal(providing_args=["xform", "tags"]) +XFORM_TAGS_DELETE = django.dispatch.Signal(providing_args=["xform", "tag"]) -@django.dispatch.receiver(xform_tags_add, sender=XForm) +# pylint: disable=unused-argument +@django.dispatch.receiver(XFORM_TAGS_ADD, sender=XForm) def add_tags_to_xform_instances(sender, **kwargs): + """Adds tags to an xform instance.""" xform = kwargs.get("xform", None) tags = kwargs.get("tags", None) if isinstance(xform, XForm) and isinstance(tags, list): @@ -20,8 +24,10 @@ def add_tags_to_xform_instances(sender, **kwargs): instance.parsed_instance.save() -@django.dispatch.receiver(xform_tags_delete, sender=XForm) +# pylint: disable=unused-argument +@django.dispatch.receiver(XFORM_TAGS_DELETE, sender=XForm) def delete_tag_from_xform_instances(sender, **kwargs): + """Deletes tags associated with an xform when it is deleted.""" xform = kwargs.get("xform", None) tag = kwargs.get("tag", None) if isinstance(xform, XForm) and isinstance(tag, str): From 6727768ebaec37cfa121accfdb4d71d9b97322a4 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 22:00:28 +0300 Subject: [PATCH 074/234] cleanup --- onadata/libs/utils/briefcase_client.py | 122 +++++++++++++++---------- 1 file changed, 73 insertions(+), 49 deletions(-) diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index ee955369c1..b4f03d1f05 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -1,3 +1,5 @@ +# -*- codingL: utf-8 -*- +"""ODK BriefcaseClient utils module""" import logging import mimetypes import os @@ -10,7 +12,6 @@ from django.core.files.storage import default_storage from django.core.files.uploadedfile import InMemoryUploadedFile from django.db import transaction -from django.utils.translation import ugettext as _ import requests from requests.auth import HTTPDigestAuth @@ -23,6 +24,7 @@ def django_file(file_obj, field_name, content_type): + """Return an InMemoryUploadedFile file object.""" return InMemoryUploadedFile( file=file_obj, field_name=field_name, @@ -47,13 +49,13 @@ def _get_form_list(xml_text): xml_doc = clean_and_parse_xml(xml_text) forms = [] - for childNode in xml_doc.childNodes: - if childNode.nodeName == "xforms": - for xformNode in childNode.childNodes: - if xformNode.nodeName == "xform": - id_string = node_value(xformNode, "formID") - download_url = node_value(xformNode, "downloadUrl") - manifest_url = node_value(xformNode, "manifestUrl") + for child_node in xml_doc.childNodes: + if child_node.nodeName == "xforms": + for xform_node in child_node.childNodes: + if xform_node.nodeName == "xform": + id_string = node_value(xform_node, "formID") + download_url = node_value(xform_node, "downloadUrl") + manifest_url = node_value(xform_node, "manifestUrl") forms.append((id_string, download_url, manifest_url)) return forms @@ -72,7 +74,9 @@ def _get_instances_uuids(xml_doc): return uuids -class BriefcaseClient(object): +class BriefcaseClient: + """ODK BriefcaseClient class""" + def __init__(self, url, username, password, user): self.url = url self.user = user @@ -85,8 +89,9 @@ def __init__(self, url, username, password, user): self.logger = logging.getLogger("console_logger") def download_manifest(self, manifest_url, id_string): + """Downloads the XForm manifest from an ODK server.""" if self._get_response(manifest_url): - manifest_res = self._current_response + manifest_res = getattr(self, "_current_response") try: manifest_doc = clean_and_parse_xml(manifest_res.content) @@ -94,63 +99,69 @@ def download_manifest(self, manifest_url, id_string): return manifest_path = os.path.join(self.forms_path, id_string, "form-media") - self.logger.debug("Downloading media files for %s" % id_string) + self.logger.debug("Downloading media files for %s", id_string) self.download_media_files(manifest_doc, manifest_path) def download_xforms(self, include_instances=False): + """Downloads the XForm XML form an ODK server.""" # fetch formList if not self._get_response(self.form_list_url): + _current_response = getattr(self, "_current_response") response = ( - self._current_response.content - if self._current_response - else "Unknown Error" + _current_response.content if _current_response else "Unknown Error" ) - self.logger.error("Failed to download xforms %s." % response) + self.logger.error("Failed to download xforms %s.", response) return - response = self._current_response + response = getattr(self, "_current_response") forms = _get_form_list(response.content) - self.logger.debug("Successfull fetched %s." % self.form_list_url) + self.logger.debug("Successfull fetched %s.", self.form_list_url) for id_string, download_url, manifest_url in forms: - form_path = os.path.join(self.forms_path, id_string, "%s.xml" % id_string) + form_path = os.path.join(self.forms_path, id_string, f"{id_string}.xml") if not default_storage.exists(form_path): if not self._get_response(download_url): - self.logger.error("Failed to download xform %s." % download_url) + self.logger.error("Failed to download xform %s.", download_url) continue - form_res = self._current_response + form_res = getattr(self, "_current_response") content = ContentFile(form_res.content.strip()) default_storage.save(form_path, content) else: form_res = default_storage.open(form_path) content = form_res.read() - self.logger.debug("Fetched %s." % download_url) + self.logger.debug("Fetched %s.", download_url) if manifest_url: self.download_manifest(manifest_url, id_string) if include_instances: self.download_instances(id_string) - self.logger.debug("Done downloading submissions for %s" % id_string) + self.logger.debug("Done downloading submissions for %s", id_string) @retry(NUM_RETRIES) def _get_response(self, url, params=None): - self._current_response = None + """ + Downloads the url and sets self._current_response with the contents. + """ + setattr(self, "_current_response", None) response = requests.get(url, auth=self.auth, params=params) success = response.status_code == 200 - self._current_response = response + setattr(self, "_current_response", response) return success @retry(NUM_RETRIES) def _get_media_response(self, url): - self._current_response = None + """ + Downloads the media file and sets self._current_response with the contents. + """ + setattr(self, "_current_response", None) head_response = requests.head(url, auth=self.auth) # S3 redirects, avoid using formhub digest on S3 @@ -159,11 +170,12 @@ def _get_media_response(self, url): response = requests.get(url) success = response.status_code == 200 - self._current_response = response + setattr(self, "_current_response", response) return success def download_media_files(self, xml_doc, media_path): + """Downloads media files from an ODK server.""" for media_node in xml_doc.getElementsByTagName("mediaFile"): filename_node = media_node.getElementsByTagName("filename") url_node = media_node.getElementsByTagName("downloadUrl") @@ -174,29 +186,35 @@ def download_media_files(self, xml_doc, media_path): continue download_url = url_node[0].childNodes[0].nodeValue if self._get_media_response(download_url): - download_res = self._current_response + download_res = getattr(self, "_current_response") media_content = ContentFile(download_res.content) default_storage.save(path, media_content) - self.logger.debug("Fetched %s." % filename) + self.logger.debug("Fetched %s.", filename) else: - self.logger.error("Failed to fetch %s." % filename) + self.logger.error("Failed to fetch %s.", filename) + # pylint: disable=too-many-locals def download_instances(self, form_id, cursor=0, num_entries=100): - self.logger.debug("Starting submissions download for %s" % form_id) + """Download the XML submissions.""" + self.logger.debug("Starting submissions download for %s", form_id) if not self._get_response( self.submission_list_url, params={"formId": form_id, "numEntries": num_entries, "cursor": cursor}, ): self.logger.error( - "Fetching %s formId: %s, cursor: %s" - % (self.submission_list_url, form_id, cursor) + "Fetching %s formId: %s, cursor: %s", + self.submission_list_url, + form_id, + cursor, ) return - response = self._current_response + response = getattr(self, "_current_response") self.logger.debug( - "Fetching %s formId: %s, cursor: %s" - % (self.submission_list_url, form_id, cursor) + "Fetching %s formId: %s, cursor: %s", + self.submission_list_url, + form_id, + cursor, ) try: xml_doc = clean_and_parse_xml(response.content) @@ -207,7 +225,7 @@ def download_instances(self, form_id, cursor=0, num_entries=100): path = os.path.join(self.forms_path, form_id, "instances") for uuid in instances: - self.logger.debug("Fetching %s %s submission" % (uuid, form_id)) + self.logger.debug("Fetching %s %s submission", uuid, form_id) form_str = ( "%(formId)s[@version=null and @uiVersion=null]/" "%(formId)s[@key=%(instanceId)s]" @@ -218,7 +236,7 @@ def download_instances(self, form_id, cursor=0, num_entries=100): if self._get_response( self.download_submission_url, params={"formId": form_str} ): - instance_res = self._current_response + instance_res = getattr(self, "_current_response") content = instance_res.content.strip() default_storage.save(instance_path, ContentFile(content)) else: @@ -234,7 +252,7 @@ def download_instances(self, form_id, cursor=0, num_entries=100): media_path = os.path.join(path, uuid.replace(":", "")) self.download_media_files(instance_doc, media_path) - self.logger.debug("Fetched %s %s submission" % (form_id, uuid)) + self.logger.debug("Fetched %s %s submission", form_id, uuid) if xml_doc.getElementsByTagName("resumptionCursor"): rs_node = xml_doc.getElementsByTagName("resumptionCursor")[0] @@ -246,6 +264,7 @@ def download_instances(self, form_id, cursor=0, num_entries=100): @transaction.atomic def _upload_xform(self, path, file_name): + """Publishes the XForm XML to the XForm model.""" xml_file = default_storage.open(path) xml_file.name = file_name k = PublishXForm(xml_file, self.user) @@ -253,6 +272,9 @@ def _upload_xform(self, path, file_name): return publish_form(k.publish_xform) def _upload_instance(self, xml_file, instance_dir_path, files): + """ + Adds an xform submission to the Instance model. + """ xml_doc = clean_and_parse_xml(xml_file.read()) xml = StringIO() de_node = xml_doc.documentElement @@ -270,7 +292,7 @@ def _upload_instance(self, xml_file, instance_dir_path, files): file_obj = default_storage.open( os.path.join(instance_dir_path, filename) ) - mimetype, encoding = mimetypes.guess_type(file_obj.name) + mimetype, _encoding = mimetypes.guess_type(file_obj.name) media_obj = django_file(file_obj, "media_files[]", mimetype) attachments.append(media_obj) @@ -278,11 +300,11 @@ def _upload_instance(self, xml_file, instance_dir_path, files): def _upload_instances(self, path): instances_count = 0 - dirs, not_in_use = default_storage.listdir(path) + dirs, _not_in_use = default_storage.listdir(path) for instance_dir in dirs: instance_dir_path = os.path.join(path, instance_dir) - i_dirs, files = default_storage.listdir(instance_dir_path) + _dirs, files = default_storage.listdir(instance_dir_path) xml_file = None if "submission.xml" in files: @@ -298,10 +320,11 @@ def _upload_instances(self, path): continue except Exception as e: logging.exception( - _( + ( "Ignoring exception, processing XML submission " - "raised exception: %s" % str(e) - ) + "raised exception: %s" + ), + str(e), ) else: instances_count += 1 @@ -309,19 +332,20 @@ def _upload_instances(self, path): return instances_count def push(self): - dirs, files = default_storage.listdir(self.forms_path) + """Publishes XForms and XForm submissions.""" + dirs, _files = default_storage.listdir(self.forms_path) for form_dir in dirs: dir_path = os.path.join(self.forms_path, form_dir) form_dirs, form_files = default_storage.listdir(dir_path) - form_xml = "%s.xml" % form_dir + form_xml = f"{form_dir}.xml" if form_xml in form_files: form_xml_path = os.path.join(dir_path, form_xml) x = self._upload_xform(form_xml_path, form_xml) if isinstance(x, dict): - self.logger.error("Failed to publish %s" % form_dir) + self.logger.error("Failed to publish %s", form_dir) else: - self.logger.debug("Successfully published %s" % form_dir) + self.logger.debug("Successfully published %s", form_dir) if "instances" in form_dirs: self.logger.debug("Uploading instances") c = self._upload_instances(os.path.join(dir_path, "instances")) - self.logger.debug("Published %d instances for %s" % (c, form_dir)) + self.logger.debug("Published %d instances for %s", c, form_dir) From 1120ec491809df93048249463e609f80d79d0687 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 29 Apr 2022 22:11:13 +0300 Subject: [PATCH 075/234] Fix xform_tags_add imports --- onadata/libs/mixins/labels_mixin.py | 54 ++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/onadata/libs/mixins/labels_mixin.py b/onadata/libs/mixins/labels_mixin.py index 99205186f3..e60d1b57c2 100644 --- a/onadata/libs/mixins/labels_mixin.py +++ b/onadata/libs/mixins/labels_mixin.py @@ -1,3 +1,5 @@ +# -*- codingL: utf-8 -*- +"""LabelMixin module""" from django import forms from rest_framework import status from rest_framework.decorators import action @@ -5,10 +7,12 @@ from taggit.forms import TagField from onadata.apps.logger.models import XForm -from onadata.libs.models.signals import xform_tags_add, xform_tags_delete +from onadata.libs.models.signals import XFORM_TAGS_ADD, XFORM_TAGS_DELETE class TagForm(forms.Form): + """TagForm form""" + tags = TagField() @@ -22,16 +26,17 @@ def _labels_post(request, instance): form = TagForm(request.data) if form.is_valid(): - tags = form.cleaned_data.get('tags', None) + tags = form.cleaned_data.get("tags", None) if tags: for tag in tags: instance.tags.add(tag) if isinstance(instance, XForm): - xform_tags_add.send(sender=XForm, xform=instance, tags=tags) + XFORM_TAGS_ADD.send(sender=XForm, xform=instance, tags=tags) return status.HTTP_201_CREATED + return None def _labels_delete(label, instance): @@ -46,11 +51,14 @@ def _labels_delete(label, instance): instance.tags.remove(label) if isinstance(instance, XForm): - xform_tags_delete.send(sender=XForm, xform=instance, tag=label) + XFORM_TAGS_DELETE.send(sender=XForm, xform=instance, tag=label) # Accepted, label does not exist hence nothing removed - http_status = status.HTTP_202_ACCEPTED \ - if count == instance.tags.names().count() else status.HTTP_200_OK + http_status = ( + status.HTTP_202_ACCEPTED + if count == instance.tags.names().count() + else status.HTTP_200_OK + ) return [http_status, list(instance.tags.names())] @@ -66,13 +74,12 @@ def process_label_request(request, label, instance): """ http_status = status.HTTP_200_OK - if request.method == 'POST': + if request.method == "POST": http_status = _labels_post(request, instance) - if request.method == 'GET' and label: - data = [tag['name'] - for tag in instance.tags.filter(name=label).values('name')] - elif request.method == 'DELETE' and label: + if request.method == "GET" and label: + data = [tag["name"] for tag in instance.tags.filter(name=label).values("name")] + elif request.method == "DELETE" and label: http_status, data = _labels_delete(label, instance) else: data = list(instance.tags.names()) @@ -80,11 +87,26 @@ def process_label_request(request, label, instance): return Response(data, status=http_status) -class LabelsMixin(object): - @action(methods=['GET', 'POST', 'DELETE'], detail=True, - extra_lookup_fields=['label', ]) - def labels(self, request, format='json', **kwargs): +# pylint: disable=too-few-public-methods +class LabelsMixin: + """LabelsMixin - adds labels method that processes labels.""" + + # pylint: disable=redefined-builtin + @action( + methods=["GET", "POST", "DELETE"], + detail=True, + extra_lookup_fields=[ + "label", + ], + ) + def labels(self, request, format="json", **kwargs): # noqa + """Process request to labels endpoint. + + :param request: HTTP request object. + + :returns: A response object based on the type of request. + """ xform = self.get_object() - label = kwargs.get('label') + label = kwargs.get("label") return process_label_request(request, label, xform) From 8c6131211a46640e629e079c822e1e7eeb263e79 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Fri, 29 Apr 2022 22:42:31 +0300 Subject: [PATCH 076/234] Update deformed f-strings Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/tests/test_briefcase_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 7f5e0e8958..14d8603f77 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -109,7 +109,7 @@ def test_download_xform_xml(self): "deno", "briefcase", "forms", self.xform.id_string ) self.assertTrue(storage.exists(forms_folder_path)) - forms_path = os.path.join(forms_folder_path, "{self.xform.id_string}.xml") + forms_path = os.path.join(forms_folder_path, f"{self.xform.id_string}.xml") self.assertTrue(storage.exists(forms_path)) form_media_path = os.path.join(forms_folder_path, "form-media") self.assertTrue(storage.exists(form_media_path)) @@ -124,12 +124,12 @@ def test_download_xform_xml(self): self.assertTrue(storage.exists(instance_folder_path)) instance = Instance.objects.all()[0] instance_path = os.path.join( - instance_folder_path, "uuid%s" % instance.uuid, "submission.xml" + instance_folder_path, f"uuid{instance.uuid}", "submission.xml" ) self.assertTrue(storage.exists(instance_path)) media_file = "1335783522563.jpg" media_path = os.path.join( - instance_folder_path, "uuid{instance.uuid}", media_file + instance_folder_path, f"uuid{instance.uuid}", media_file ) self.assertTrue(storage.exists(media_path)) From 640bc52c1e1ce250ee67e1ffea86098fc6be9ca9 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 30 Apr 2022 00:16:04 +0300 Subject: [PATCH 077/234] batch: cleanup --- onadata/libs/filters.py | 74 +++++++++++++++---- onadata/libs/mixins/labels_mixin.py | 2 +- onadata/libs/models/signals.py | 2 +- onadata/libs/serializers/fields/json_field.py | 25 +++++-- onadata/libs/utils/briefcase_client.py | 26 ++++--- onadata/libs/utils/decorators.py | 9 ++- onadata/libs/utils/gravatar.py | 6 +- onadata/libs/utils/qrcode.py | 18 ++--- 8 files changed, 115 insertions(+), 47 deletions(-) diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 1e00df7cbb..37b12c8450 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,7 +1,8 @@ -import six - +# -*- coding: utf-8 -*- from uuid import UUID +import six + from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist @@ -20,10 +21,23 @@ from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms +# pylint: disable=invalid-name User = get_user_model() +def _is_public_xform(export_id: int): + export = Export.objects.filter(pk=export_id).first() + + if export: + return export.xform.shared_data or export.xform.shared + + return False + + +# pylint: disable=too-few-public-methods class AnonDjangoObjectPermissionFilter(ObjectPermissionsFilter): + """Anonymous user permission filter class.""" + def filter_queryset(self, request, queryset, view): """ Anonymous user has no object permissions, return queryset as it is. @@ -71,18 +85,25 @@ class EnketoAnonDjangoObjectPermissionFilter(AnonDjangoObjectPermissionFilter): def filter_queryset(self, request, queryset, view): """Check report_xform permission when requesting for Enketo URL.""" if view.action == "enketo": - self.perm_format = ( - "%(app_label)s.report_%(model_name)s" # noqa pylint: disable=W0201 - ) + # noqa pylint: disable=attribute-defined-outside-init + self.perm_format = "%(app_label)s.report_%(model_name)s" + return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): + """XFormList permission filter with using [app].report_[model] form.""" + perm_format = "%(app_label)s.report_%(model_name)s" class XFormListXFormPKFilter: + """Filter forms via 'xform_pk' param.""" + + # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): + """Returns an XForm queryset filtered by the 1xform_pk' param.""" xform_pk = view.kwargs.get("xform_pk") if xform_pk: try: @@ -98,6 +119,7 @@ def filter_queryset(self, request, queryset, view): class FormIDFilter(django_filter_filters.FilterSet): + """formID filter using the XForm.id_string.""" formID = django_filter_filters.CharFilter(field_name="id_string") # noqa @@ -106,7 +128,13 @@ class Meta: fields = ["formID"] +# pylint: disable=too-few-public-methods class OrganizationPermissionFilter(ObjectPermissionsFilter): + """Organization profiles filter + + Based on the organization the profile is added to. + """ + def filter_queryset(self, request, queryset, view): """Return a filtered queryset or all profiles if a getting a specific profile.""" @@ -122,6 +150,7 @@ def filter_queryset(self, request, queryset, view): return queryset.model.objects.filter(user__in=org_users, user__is_active=True) +# pylint: disable=too-few-public-methods class XFormOwnerFilter(filters.BaseFilterBackend): owner_prefix = "user" @@ -137,6 +166,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class DataFilter(ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): if request.user.is_anonymous: @@ -200,6 +230,7 @@ class Meta: } +# pylint: disable=too-few-public-methods class ProjectOwnerFilter(filters.BaseFilterBackend): owner_prefix = "organization" @@ -216,6 +247,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class AnonUserProjectFilter(ObjectPermissionsFilter): owner_prefix = "organization" @@ -247,6 +279,7 @@ def filter_queryset(self, request, queryset, view): return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class TagFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): # filter by tags if available. @@ -259,6 +292,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class XFormPermissionFilterMixin: def _xform_filter(self, request, view, keyword): """Use XForm permissions""" @@ -287,6 +321,7 @@ def _xform_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) +# pylint: disable=too-few-public-methods class ProjectPermissionFilterMixin: def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") @@ -313,6 +348,7 @@ def _project_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) +# pylint: disable=too-few-public-methods class InstancePermissionFilterMixin: # pylint: disable=too-many-locals def _instance_filter(self, request, view, keyword): @@ -363,11 +399,13 @@ def _instance_filter_queryset(self, request, queryset, view, keyword): return queryset.filter(**kwarg) +# pylint: disable=too-few-public-methods class RestServiceFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): return self._xform_filter_queryset(request, queryset, view, "xform_id") +# pylint: disable=too-few-public-methods class MetaDataFilter( ProjectPermissionFilterMixin, InstancePermissionFilterMixin, @@ -413,6 +451,7 @@ def filter_queryset(self, request, queryset, view): ) +# pylint: disable=too-few-public-methods class AttachmentFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): @@ -440,6 +479,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class AttachmentTypeFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): attachment_type = request.query_params.get("type") @@ -452,6 +492,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class TeamOrgFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): org = request.data.get("org") or request.query_params.get("org") @@ -465,6 +506,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class UserNoOrganizationsFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): if str(request.query_params.get("orgs")).lower() == "false": @@ -476,6 +518,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class OrganizationsSharedWithUserFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): """ @@ -508,6 +551,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class WidgetFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): def filter_queryset(self, request, queryset, view): @@ -518,6 +562,7 @@ def filter_queryset(self, request, queryset, view): return super().filter_queryset(request, queryset, view) +# pylint: disable=too-few-public-methods class UserProfileFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): if view.action == "list": @@ -533,6 +578,7 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class NoteFilter(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): instance_id = request.query_params.get("instance") @@ -549,26 +595,19 @@ def filter_queryset(self, request, queryset, view): return queryset +# pylint: disable=too-few-public-methods class ExportFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): """ ExportFilter class uses permissions on the related xform to filter Export queryesets. Also filters submitted_by a specific user. """ - def _is_public_xform(self, export_id: int): - export = Export.objects.filter(pk=export_id).first() - - if export: - return export.xform.shared_data or export.xform.shared - - return False - def filter_queryset(self, request, queryset, view): has_submitted_by_key = ( Q(options__has_key="query") & Q(options__query__has_key="_submitted_by"), ) - if request.user.is_anonymous or self._is_public_xform(view.kwargs.get("pk")): + if request.user.is_anonymous or _is_public_xform(view.kwargs.get("pk")): return self._xform_filter_queryset( request, queryset, view, "xform_id" ).exclude(*has_submitted_by_key) @@ -576,6 +615,7 @@ def filter_queryset(self, request, queryset, view): old_perm_format = self.perm_format # only if request.user has access to all data + # noqa pylint: disable=attribute-defined-outside-init self.perm_format = old_perm_format + "_all" all_qs = self._xform_filter_queryset( request, queryset, view, "xform_id" @@ -592,9 +632,13 @@ def filter_queryset(self, request, queryset, view): return all_qs | submitter_qs +# pylint: disable=too-few-public-methods class PublicDatasetsFilter: - # pylint: disable=unused-argument + """Public data set filter where the share attribute is True""" + + # pylint: disable=unused-argument,no-self-use def filter_queryset(self, request, queryset, view): + """Return a queryset of shared=True data if the user is anonymous.""" if request and request.user.is_anonymous: return queryset.filter(shared=True) diff --git a/onadata/libs/mixins/labels_mixin.py b/onadata/libs/mixins/labels_mixin.py index e60d1b57c2..57648b23c9 100644 --- a/onadata/libs/mixins/labels_mixin.py +++ b/onadata/libs/mixins/labels_mixin.py @@ -1,4 +1,4 @@ -# -*- codingL: utf-8 -*- +# -*- coding: utf-8 -*- """LabelMixin module""" from django import forms from rest_framework import status diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index a52ea40626..2fb10a693b 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -1,4 +1,4 @@ -# -*- codingL: utf-8 -*- +# -*- coding: utf-8 -*- """onadata.libs.models.signals module""" import django.dispatch diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index 75f54d3f5e..e0c010f251 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -1,25 +1,38 @@ +# -*- coding: utf-8 -*- +""" +A string is represented as valid JSON and is accessible as a dictionary and vis-a-vis. +""" import json -from builtins import str as text + from rest_framework import serializers class JsonField(serializers.Field): + """ + Deserialize a string instance containing a JSON document to a Python object. + """ + def to_representation(self, value): + """ + Deserialize ``value`` a `str` instance containing a + JSON document to a Python object. + """ if isinstance(value, str): return json.loads(value) return value - def to_internal_value(self, value): - if isinstance(value, str): + def to_internal_value(self, data): + if isinstance(data, str): try: - return json.loads(value) + return json.loads(data) except ValueError as e: # invalid json - raise serializers.ValidationError(text(e)) - return value + raise serializers.ValidationError(str(e)) + return data @classmethod def to_json(cls, data): + """Returns the JSON string as a dictionary.""" if isinstance(data, str): return json.loads(data) return data diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index b4f03d1f05..a928a7c7b2 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -1,4 +1,4 @@ -# -*- codingL: utf-8 -*- +# -*- coding: utf-8 -*- """ODK BriefcaseClient utils module""" import logging import mimetypes @@ -197,10 +197,11 @@ def download_media_files(self, xml_doc, media_path): def download_instances(self, form_id, cursor=0, num_entries=100): """Download the XML submissions.""" self.logger.debug("Starting submissions download for %s", form_id) - if not self._get_response( + downloaded = self._get_response( self.submission_list_url, params={"formId": form_id, "numEntries": num_entries, "cursor": cursor}, - ): + ) + if not downloaded: self.logger.error( "Fetching %s formId: %s, cursor: %s", self.submission_list_url, @@ -233,9 +234,10 @@ def download_instances(self, form_id, cursor=0, num_entries=100): ) instance_path = os.path.join(path, uuid.replace(":", ""), "submission.xml") if not default_storage.exists(instance_path): - if self._get_response( + downloaded = self._get_response( self.download_submission_url, params={"formId": form_str} - ): + ) + if downloaded: instance_res = getattr(self, "_current_response") content = instance_res.content.strip() default_storage.save(instance_path, ContentFile(content)) @@ -318,7 +320,9 @@ def _upload_instances(self, path): self._upload_instance(xml_file, instance_dir_path, files) except ExpatError: continue + # pylint: disable=broad-except except Exception as e: + # keep going despite some errors. logging.exception( ( "Ignoring exception, processing XML submission " @@ -340,12 +344,16 @@ def push(self): form_xml = f"{form_dir}.xml" if form_xml in form_files: form_xml_path = os.path.join(dir_path, form_xml) - x = self._upload_xform(form_xml_path, form_xml) - if isinstance(x, dict): + published_xform = self._upload_xform(form_xml_path, form_xml) + if isinstance(published_xform, dict): self.logger.error("Failed to publish %s", form_dir) else: self.logger.debug("Successfully published %s", form_dir) if "instances" in form_dirs: self.logger.debug("Uploading instances") - c = self._upload_instances(os.path.join(dir_path, "instances")) - self.logger.debug("Published %d instances for %s", c, form_dir) + submission_count = self._upload_instances( + os.path.join(dir_path, "instances") + ) + self.logger.debug( + "Published %d instances for %s", submission_count, form_dir + ) diff --git a/onadata/libs/utils/decorators.py b/onadata/libs/utils/decorators.py index 43d3518fc0..faa39d7814 100644 --- a/onadata/libs/utils/decorators.py +++ b/onadata/libs/utils/decorators.py @@ -10,6 +10,7 @@ def check_obj(f): + """Checks if the first argument is truthy and then calls the underlying function.""" # pylint: disable=inconsistent-return-statements @wraps(f) def with_check_obj(*args, **kwargs): @@ -20,6 +21,8 @@ def with_check_obj(*args, **kwargs): def is_owner(view_func): + """Redirects to login if not owner.""" + @wraps(view_func) def _wrapped_view(request, *args, **kwargs): # assume username is first arg @@ -34,9 +37,9 @@ def _wrapped_view(request, *args, **kwargs): # use the path as the "next" url. login_scheme, login_netloc = urlparse(login_url)[:2] current_scheme, current_netloc = urlparse(path)[:2] - if (not login_scheme or login_scheme == current_scheme) and ( - not login_netloc or login_netloc == current_netloc - ): + is_scheme = not login_scheme or login_scheme == current_scheme + is_netloc = not login_netloc or login_netloc == current_netloc + if is_scheme and is_netloc: path = request.get_full_path() return redirect_to_login(path, None, REDIRECT_FIELD_NAME) diff --git a/onadata/libs/utils/gravatar.py b/onadata/libs/utils/gravatar.py index a89ec236f3..f508e32ed0 100644 --- a/onadata/libs/utils/gravatar.py +++ b/onadata/libs/utils/gravatar.py @@ -1,9 +1,11 @@ -# -*- codingL: utf-8 -*- +# -*- coding: utf-8 -*- """Gravatar utils module""" import hashlib -import requests + from six.moves.urllib.parse import urlencode +import requests + DEFAULT_GRAVATAR = "https://ona.io/static/images/default_avatar.png" GRAVATAR_ENDPOINT = "https://secure.gravatar.com/avatar/" diff --git a/onadata/libs/utils/qrcode.py b/onadata/libs/utils/qrcode.py index 9a104e55e0..0f13a4c238 100644 --- a/onadata/libs/utils/qrcode.py +++ b/onadata/libs/utils/qrcode.py @@ -2,22 +2,20 @@ """ QR code utility function. """ +from io import BytesIO from base64 import b64encode from elaphe import barcode -from io import BytesIO # pylint: disable=too-many-arguments -def generate_qrcode( - message, - stream=None, - eclevel="M", - margin=10, - data_mode="8bits", - image_format="PNG", - scale=2.5, -): +def generate_qrcode(message): """Generate a QRCode, settings options and output.""" + stream = None + eclevel = "M" + margin = 10 + data_mode = "8bits" + image_format = "PNG" + scale = 2.5 if stream is None: stream = BytesIO() From 50baa77b3d3cf79981eb1adfd8053ee06d5acb6b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 1 May 2022 10:32:53 +0300 Subject: [PATCH 078/234] batch: cleanup --- docker/docker-entrypoint.sh | 2 +- onadata/apps/api/models/team.py | 25 ++- .../api/tests/viewsets/test_xform_viewset.py | 4 +- .../management/commands/import_instances.py | 41 ++-- onadata/apps/main/models/audit.py | 98 ++++++---- onadata/libs/authentication.py | 110 ++++++----- .../libs/serializers/attachment_serializer.py | 30 ++- onadata/libs/serializers/fields/json_field.py | 6 + onadata/libs/serializers/widget_serializer.py | 17 +- onadata/libs/serializers/xform_serializer.py | 183 +++++++++++++----- onadata/libs/utils/email.py | 17 +- 11 files changed, 356 insertions(+), 177 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 367a2b1cb5..446e5e9400 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,7 +13,7 @@ if $RUN_DB_INIT_SCRIPT; then psql -h db -U postgres onadata -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" fi -virtualenv -p "$(which ${SELECTED_PYTHON})" "/srv/onadata/.virtualenv/${SELECTED_PYTHON}" +virtualenv -p "$(which "${SELECTED_PYTHON}")" "/srv/onadata/.virtualenv/${SELECTED_PYTHON}" . /srv/onadata/.virtualenv/"${SELECTED_PYTHON}"/bin/activate cd /srv/onadata diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index a96bde6f3f..ad4a8e21ce 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -1,17 +1,23 @@ +# -*- coding: utf-8 -*- +""" +Team model +""" from django.db import models from django.db.models.signals import post_save -from django.contrib.auth.models import User, Group +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from guardian.shortcuts import assign_perm, get_perms_for_model from onadata.apps.logger.models.project import Project +# pylint: disable=invalid-name +User = get_user_model() + + class Team(Group): """ - TODO: documentation - TODO: Whenever a member is removed from members team, - we should remove them from all teams and projects - within the organization. + Team model based on the Group. """ class Meta: @@ -38,17 +44,22 @@ def __str__(self): @property def team_name(self): + """Return the team name.""" return self.__str__() def save(self, *args, **kwargs): # allow use of same name in different organizations/users # concat with # if not self.name.startswith("#".join([self.organization.username])): - self.name = "%s#%s" % (self.organization.username, self.name) - super(Team, self).save(*args, **kwargs) + self.name = f"{self.organization.username}#{self.name}" + super().save(*args, **kwargs) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """ + Apply permissions to the creator of the team. + """ if created: for perm in get_perms_for_model(Team): assign_perm(perm.codename, instance.organization, instance) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b47abde86f..4ca8c45f8f 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -4448,7 +4448,7 @@ def test_multitple_enketo_urls(self): self.assertEquals(2, count) # delete cache - safe_delete("{}{}".format(ENKETO_URL_CACHE, self.xform.pk)) + safe_delete(f"{ENKETO_URL_CACHE}{self.xform.pk}") view = XFormViewSet.as_view( { @@ -5076,7 +5076,7 @@ def test_xform_version_count(self): instance.set_deleted() # delete cache - safe_delete("{}{}".format(XFORM_DATA_VERSIONS, self.xform.pk)) + safe_delete(f"{XFORM_DATA_VERSIONS}{self.xform.pk}") request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk) diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index ab12285dd6..f60d8191fd 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -1,9 +1,12 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=5 coding=utf-8 +""" +import_instances - import ODK instances from a zipped file. +""" import os -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy @@ -13,8 +16,15 @@ import_instances_from_zip, ) +# pylint: disable=invalid-name +User = get_user_model() + class Command(BaseCommand): + """ + import_instances - import ODK instances from a zipped file. + """ + args = "username path" help = ugettext_lazy( "Import a zip file, a directory containing zip files " @@ -36,26 +46,29 @@ def handle(self, *args, **kwargs): raise CommandError(_("Usage: username file/path.")) username = args[0] path = args[1] - is_async = args[2] if len(args) > 2 else False - is_async = ( - True if isinstance(is_async, str) and is_async.lower() == "true" else False - ) + is_async = False + if len(args) > 2: + if isinstance(args[2], str): + is_async = args[2].lower() == "true" + try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The specified user '%s' does not exist.") % username) + except User.DoesNotExist as e: + raise CommandError( + _(f"The specified user '{username}' does not exist.") + ) from e # make sure path exists if not os.path.exists(path): - raise CommandError(_("The specified path '%s' does not exist.") % path) + raise CommandError(_(f"The specified path '{path}' does not exist.")) - for dir, subdirs, files in os.walk(path): - # check if the dir has an odk directory + for directory, subdirs, files in os.walk(path): + # check if the directory has an odk directory if "odk" in subdirs: - # dont walk further down this dir + # dont walk further down this directory subdirs.remove("odk") - self.stdout.write(_("Importing from dir %s..\n") % dir) - results = import_instances_from_path(dir, user, is_async=is_async) + self.stdout.write(_(f"Importing from directory {directory}..\n")) + results = import_instances_from_path(directory, user, is_async=is_async) self._log_import(results) for file in files: filepath = os.path.join(path, file) @@ -63,6 +76,6 @@ def handle(self, *args, **kwargs): os.path.isfile(filepath) and os.path.splitext(filepath)[1].lower() == ".zip" ): - self.stdout.write(_("Importing from zip at %s..\n") % filepath) + self.stdout.write(_(f"Importing from zip at {filepath}..\n")) results = import_instances_from_zip(filepath, user) self._log_import(results) diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 445d7054a9..208c0192ce 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Audit model +""" import json import six @@ -13,25 +17,32 @@ class Audit(models.Model): json = JSONField() class Meta: - app_label = 'main' + app_label = "main" -class AuditLog(object): - ACCOUNT = u"account" +class AuditLog: + """ + AuditLog - creates and provide access to the Audit model records. + """ + + ACCOUNT = "account" DEFAULT_BATCHSIZE = 1000 - CREATED_ON = u"created_on" + CREATED_ON = "created_on" def __init__(self, data): self.data = data def save(self): - a = Audit(json=self.data) - a.save() + audit = Audit(json=self.data) + audit.save() - return a + return audit @classmethod - def query_iterator(cls, sql, fields=None, params=[], count=False): + def query_iterator(cls, sql, fields=None, params=None, count=False): + """ + Returns an iterator of all records. + """ # cursor seems to stringify dicts # added workaround to parse stringified dicts to json def parse_json(data): @@ -40,20 +51,21 @@ def parse_json(data): except ValueError: return data + params = params if params is not None else [] cursor = connection.cursor() sql_params = fields + params if fields is not None else params if count: - from_pos = sql.upper().find(' FROM') + from_pos = sql.upper().find(" FROM") if from_pos != -1: - sql = u"SELECT COUNT(*) " + sql[from_pos:] + sql = "SELECT COUNT(*) " + sql[from_pos:] - order_pos = sql.upper().find('ORDER BY') + order_pos = sql.upper().find("ORDER BY") if order_pos != -1: sql = sql[:order_pos] sql_params = params - fields = [u'count'] + fields = ["count"] cursor.execute(sql, sql_params) @@ -64,52 +76,62 @@ def parse_json(data): for row in cursor.fetchall(): yield dict(zip(fields, row)) + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments @classmethod - def query_data(cls, username, query=None, fields=None, sort=None, start=0, - limit=DEFAULT_LIMIT, count=False): + def query_data( + cls, + username, + query=None, + fields=None, + sort=None, + start=0, + limit=DEFAULT_LIMIT, + count=False, + ): if start is not None and (start < 0 or limit < 0): raise ValueError(_("Invalid start/limit params")) - sort = 'pk' if sort is None else sort - instances = Audit.objects.filter().extra(where=["json->>%s = %s"], - params=['account', username]) + sort = "pk" if sort is None else sort + instances = Audit.objects.filter().extra( + where=["json->>%s = %s"], params=["account", username] + ) where_params = [] - sql_where = u"" + sql_where = "" if query and isinstance(query, six.string_types): query = json.loads(query) or_where = [] or_params = [] - if '$or' in list(query): - or_dict = query.pop('$or') + if "$or" in list(query): + or_dict = query.pop("$or") for or_query in or_dict: - or_where.extend( - [u"json->>%s = %s" for i in or_query.items()]) + or_where.extend(["json->>%s = %s" for i in or_query.items()]) [ # pylint: disable=expression-not-assigned - or_params.extend(i) for i in or_query.items()] + or_params.extend(i) for i in or_query.items() + ] - or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] + or_where = ["".join(["(", " OR ".join(or_where), ")"])] - where = [u"json->>%s = %s" for i in query.items()] + or_where - [where_params.extend(i) for i in query.items()] + where = ["json->>%s = %s" for i in query.items()] + or_where + for i in query.items(): + where_params.extend(i) where_params.extend(or_params) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) if fields: - field_list = [u"json->%s" for i in fields] - sql = u"SELECT %s FROM main_audit" % u",".join(field_list) + field_list = ["json->%s" for i in fields] + sql = f"SELECT {','.join(field_list)} FROM main_audit" if where_params: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE json->>%s = %s " + sql_where \ - + u" ORDER BY id" - params = ['account', username] + where_params + sql += " WHERE json->>%s = %s " + sql_where + " ORDER BY id" + params = ["account", username] + where_params if start is not None: - sql += u" OFFSET %s LIMIT %s" + sql += " OFFSET %s LIMIT %s" params += [start, limit] records = cls.query_iterator(sql, fields, params, count) else: @@ -119,21 +141,21 @@ def query_data(cls, username, query=None, fields=None, sort=None, start=0, if where_params: instances = instances.extra(where=where, params=where_params) - records = instances.values_list('json', flat=True) + records = instances.values_list("json", flat=True) sql, params = records.query.sql_with_params() if isinstance(sort, six.string_types) and len(sort) > 0: - direction = 'DESC' if sort.startswith('-') else 'ASC' - sort = sort[1:] if sort.startswith('-') else sort - sql = u'{} ORDER BY json->>%s {}'.format(sql, direction) + direction = "DESC" if sort.startswith("-") else "ASC" + sort = sort[1:] if sort.startswith("-") else sort + sql = f"{sql} ORDER BY json->>%s {direction}" params += (sort,) if start is not None: # some inconsistent/weird behavior I noticed with django's # queryset made me have to do a raw query # records = records[start: limit] - sql = u"{} OFFSET %s LIMIT %s".format(sql) + sql = f"{sql} OFFSET %s LIMIT %s" params += (start, limit) records = cls.query_iterator(sql, None, list(params)) diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index d79bc549c2..1f14520422 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -7,7 +7,8 @@ from typing import Optional, Tuple from django.conf import settings -from django.contrib.auth.models import User, update_last_login +from django.contrib.auth import get_user_model +from django.contrib.auth.models import update_last_login from django.core.signing import BadSignature from django.db import DataError from django.utils import timezone @@ -17,6 +18,8 @@ from django_digest import HttpDigestAuthenticator from multidb.pinning import use_master from oauth2_provider.models import AccessToken +from oauth2_provider.oauth2_validators import OAuth2Validator +from oauth2_provider.settings import oauth2_settings from rest_framework import exceptions from rest_framework.authentication import ( BaseAuthentication, @@ -25,8 +28,6 @@ ) from rest_framework.authtoken.models import Token from rest_framework.exceptions import AuthenticationFailed -from oauth2_provider.oauth2_validators import OAuth2Validator -from oauth2_provider.settings import oauth2_settings from oidc.utils import authenticate_sso from onadata.apps.api.models.temp_token import TempToken @@ -61,6 +62,9 @@ ], ) +# pylint: disable=invalid-name +User = get_user_model() + def expired(time_token_created): """Checks if the time between when time_token_created and current time @@ -72,7 +76,7 @@ def expired(time_token_created): time_diff = (timezone.now() - time_token_created).total_seconds() token_expiry_time = TEMP_TOKEN_EXPIRY_TIME - return True if time_diff > token_expiry_time else False + return time_diff > token_expiry_time def get_api_token(cookie_jwt): @@ -86,11 +90,11 @@ def get_api_token(cookie_jwt): api_token = Token.objects.get(key=jwt_payload.get(API_TOKEN)) return api_token except BadSignature as e: - raise exceptions.AuthenticationFailed(_("Bad Signature: %s" % e)) + raise exceptions.AuthenticationFailed(_(f"Bad Signature: {e}")) from e except jwt.DecodeError as e: - raise exceptions.AuthenticationFailed(_("JWT DecodeError: %s" % e)) - except Token.DoesNotExist: - raise exceptions.AuthenticationFailed(_("Invalid token")) + raise exceptions.AuthenticationFailed(_("JWT DecodeError: {e}")) from e + except Token.DoesNotExist as e: + raise exceptions.AuthenticationFailed(_("Invalid token")) from e class DigestAuthentication(BaseAuthentication): @@ -110,24 +114,20 @@ def authenticate(self, request): if self.authenticator.authenticate(request): update_last_login(None, request.user) return request.user, None - else: - attempts = login_attempts(request) - remaining_attempts = ( - getattr(settings, "MAX_LOGIN_ATTEMPTS", 10) - attempts - ) - raise AuthenticationFailed( - _( - "Invalid username/password. " - "For security reasons, after {} more failed " - "login attempts you'll have to wait {} minutes " - "before trying again.".format( - remaining_attempts, - getattr(settings, "LOCKOUT_TIME", 1800) // 60, - ) - ) + attempts = login_attempts(request) + remaining_attempts = getattr(settings, "MAX_LOGIN_ATTEMPTS", 10) - attempts + # pylint: disable=unused-variable + lockout_time = getattr(settings, "LOCKOUT_TIME", 1800) // 60 # noqa + raise AuthenticationFailed( + _( + "Invalid username/password. " + f"For security reasons, after {remaining_attempts} more failed " + "login attempts you'll have to wait {lockout_time} minutes " + "before trying again." ) + ) except (AttributeError, ValueError, DataError) as e: - raise AuthenticationFailed(e) + raise AuthenticationFailed(e) from e def authenticate_header(self, request): response = self.authenticator.build_challenge_response() @@ -149,7 +149,7 @@ def authenticate(self, request): if len(auth) == 1: error_msg = _("Invalid token header. No credentials provided.") raise exceptions.AuthenticationFailed(error_msg) - elif len(auth) > 2: + if len(auth) > 2: error_msg = _( "Invalid token header. " "Token string should not contain spaces." ) @@ -162,7 +162,7 @@ def authenticate_credentials(self, key): if isinstance(key, bytes): key = key.decode("utf-8") token = self.model.objects.get(key=key) - except self.model.DoesNotExist: + except self.model.DoesNotExist as e: invalid_token = True if getattr(settings, "SLAVE_DATABASES", []): try: @@ -173,7 +173,7 @@ def authenticate_credentials(self, key): else: invalid_token = False if invalid_token: - raise exceptions.AuthenticationFailed(_("Invalid token")) + raise exceptions.AuthenticationFailed(_("Invalid token")) from e if not token.user.is_active: raise exceptions.AuthenticationFailed(_("User inactive or deleted")) @@ -201,11 +201,11 @@ def authenticate(self, request): if getattr(api_token, "user"): return api_token.user, api_token - except self.model.DoesNotExist: - raise exceptions.AuthenticationFailed(_("Invalid token")) + except self.model.DoesNotExist as e: + raise exceptions.AuthenticationFailed(_("Invalid token")) from e except KeyError: pass - except BadSignature: + except BadSignature as e: # if the cookie wasn't signed it means zebra might have # generated it cookie_jwt = request.COOKIES.get(ENKETO_AUTH_COOKIE) @@ -215,7 +215,7 @@ def authenticate(self, request): raise exceptions.ParseError( _("Malformed cookie. Clear your cookies then try again") - ) + ) from e return None @@ -244,12 +244,15 @@ def authenticate(self, request): # pylint: disable=no-self-use def retrieve_user_identification(request) -> Tuple[Optional[str], Optional[str]]: - ip = None + """ + Retrieve user information from a HTTP request. + """ + ip_address = None if request.META.get("HTTP_X_REAL_IP"): - ip = request.META["HTTP_X_REAL_IP"].split(",")[0] + ip_address = request.META["HTTP_X_REAL_IP"].split(",")[0] else: - ip = request.META.get("REMOTE_ADDR") + ip_address = request.META.get("REMOTE_ADDR") try: if isinstance(request.META["HTTP_AUTHORIZATION"], bytes): @@ -263,7 +266,7 @@ def retrieve_user_identification(request) -> Tuple[Optional[str], Optional[str]] else: if not username: raise AuthenticationFailed(_("Invalid username")) - return ip, username + return ip_address, username return None, None @@ -276,10 +279,10 @@ def check_lockout(request) -> Tuple[Optional[str], Optional[str]]: """ uri_path = request.get_full_path() if not any(part in LOCKOUT_EXCLUDED_PATHS for part in uri_path.split("/")): - ip, username = retrieve_user_identification(request) + ip_address, username = retrieve_user_identification(request) - if ip and username: - lockout = cache.get(safe_key("{}{}-{}".format(LOCKOUT_IP, ip, username))) + if ip_address and username: + lockout = cache.get(safe_key(f"{LOCKOUT_IP}{ip_address}-{username}")) if lockout: time_locked_out = datetime.now() - datetime.strptime( lockout, "%Y-%m-%dT%H:%M:%S" @@ -290,12 +293,11 @@ def check_lockout(request) -> Tuple[Optional[str], Optional[str]]: ) raise AuthenticationFailed( _( - "Locked out. Too many wrong username" - "/password attempts. " - "Try again in {} minutes.".format(remaining_time) + "Locked out. Too many wrong username/password attempts. " + f"Try again in {remaining_time} minutes." ) ) - return ip, username + return ip_address, username return None, None @@ -304,18 +306,18 @@ def login_attempts(request): Track number of login attempts made by a specific IP within a specified amount of time """ - ip, username = check_lockout(request) - attempts_key = safe_key("{}{}-{}".format(LOGIN_ATTEMPTS, ip, username)) + ip_address, username = check_lockout(request) + attempts_key = safe_key(f"{LOGIN_ATTEMPTS}{ip_address}-{username}") attempts = cache.get(attempts_key) if attempts: cache.incr(attempts_key) attempts = cache.get(attempts_key) if attempts >= getattr(settings, "MAX_LOGIN_ATTEMPTS", 10): - lockout_key = safe_key("{}{}-{}".format(LOCKOUT_IP, ip, username)) + lockout_key = safe_key(f"{LOCKOUT_IP}{ip_address}-{username}") lockout = cache.get(lockout_key) if not lockout: - send_lockout_email(username, ip) + send_lockout_email(username, ip_address) cache.set( lockout_key, datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), @@ -330,14 +332,17 @@ def login_attempts(request): return cache.get(attempts_key) -def send_lockout_email(username, ip): +def send_lockout_email(username, ip_address): + """ + Send locked out email + """ try: user = User.objects.get(username=username) except User.DoesNotExist: pass else: - email_data = get_account_lockout_email_data(username, ip) - end_email_data = get_account_lockout_email_data(username, ip, end=True) + email_data = get_account_lockout_email_data(username, ip_address) + end_email_data = get_account_lockout_email_data(username, ip_address, end=True) send_account_lockout_email.apply_async( args=[ @@ -404,6 +409,7 @@ def validate_bearer_token(self, token, scopes, request): request.scopes = scopes request.access_token = access_token return True - else: - self._set_oauth2_error_on_request(request, access_token, scopes) - return False + + self._set_oauth2_error_on_request(request, access_token, scopes) + + return False diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index 9459353635..a1c4b6d6a9 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -1,4 +1,9 @@ +# -*- coding: utf-8 -*- +""" +Attachments serializer. +""" import json + from six import itervalues from rest_framework import serializers @@ -28,18 +33,21 @@ def get_path(data, question_name, path_list): name = data.get("name") if name == question_name: return "/".join(path_list) - elif data.get("children") is not None: + if data.get("children") is not None: for node in data.get("children"): path_list.append(node.get("name")) path = get_path(node, question_name, path_list) if path is not None: return path - else: - del path_list[len(path_list) - 1] + del path_list[len(path_list) - 1] return None class AttachmentSerializer(serializers.HyperlinkedModelSerializer): + """ + Attachments serializer + """ + url = serializers.HyperlinkedIdentityField( view_name="attachment-detail", lookup_field="pk" ) @@ -68,30 +76,46 @@ class Meta: @check_obj def get_download_url(self, obj): + """ + Return attachment download url. + """ request = self.context.get("request") if obj: path = get_attachment_url(obj) return request.build_absolute_uri(path) if request else path + return "" def get_small_download_url(self, obj): + """ + Return attachment download url for resized small image. + """ request = self.context.get("request") if obj.mimetype.startswith("image"): path = get_attachment_url(obj, "small") return request.build_absolute_uri(path) if request else path + return "" def get_medium_download_url(self, obj): + """ + Return attachment download url for resized medium image. + """ request = self.context.get("request") if obj.mimetype.startswith("image"): path = get_attachment_url(obj, "medium") return request.build_absolute_uri(path) if request else path + return "" + # pylint: disable=no-self-use def get_field_xpath(self, obj): + """ + Return question xpath + """ qa_dict = obj.instance.get_dict() if obj.filename not in qa_dict.values(): return None diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index e0c010f251..d247f84f13 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -12,6 +12,7 @@ class JsonField(serializers.Field): Deserialize a string instance containing a JSON document to a Python object. """ + # pylint: disable=no-self-use def to_representation(self, value): """ Deserialize ``value`` a `str` instance containing a @@ -21,7 +22,12 @@ def to_representation(self, value): return json.loads(value) return value + # pylint: disable=no-self-use def to_internal_value(self, data): + """ + Deserialize ``value`` a `str` instance containing a + JSON document to a Python object. + """ if isinstance(data, str): try: return json.loads(data) diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index 5e99fa0ffa..f5a8ef1cd5 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Widget serializer +""" from django.core.exceptions import ObjectDoesNotExist from django.http import Http404 from django.urls import resolve, get_script_prefix, Resolver404 @@ -27,6 +31,7 @@ def __init__(self, *args, **kwargs): self.resolve = resolve self.reverse = reverse self.format = kwargs.pop("format", "json") + # pylint: disable=bad-super-call super(serializers.RelatedField, self).__init__(*args, **kwargs) def _setup_field(self, view_name): @@ -48,6 +53,7 @@ def to_representation(self, value): self._setup_field(self.view_name) + # pylint: disable=bad-super-call,super-with-arguments return super(GenericRelatedField, self).to_representation(value) def to_internal_value(self, data): @@ -82,6 +88,11 @@ def to_internal_value(self, data): class WidgetSerializer(serializers.HyperlinkedModelSerializer): + """ + WidgetSerializer + """ + + # pylint: disable=invalid-name id = serializers.ReadOnlyField() url = serializers.HyperlinkedIdentityField( view_name="widgets-detail", lookup_field="pk" @@ -142,10 +153,10 @@ def validate(self, attrs): try: # Check if column exists in xform get_field_from_field_xpath(column, xform) - except Http404: + except Http404 as e: raise serializers.ValidationError( - {"column": ("'{}' not in the form.".format(column))} - ) + {"column": f"'{column}' not in the form."} + ) from e order = attrs.get("order") diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 929e10011f..5824806a49 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +XForm model serialization. +""" import logging import os from hashlib import md5 @@ -5,7 +9,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.serialization import load_pem_public_key from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.core.exceptions import ValidationError from django.core.validators import URLValidator @@ -13,7 +17,6 @@ from django.utils.translation import ugettext as _ from six.moves.urllib.parse import urlparse from six import itervalues -from requests.exceptions import ConnectionError from rest_framework import serializers from rest_framework.reverse import reverse @@ -44,6 +47,9 @@ settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 ) +# pylint: disable=invalid-name +User = get_user_model() + def _create_enketo_urls(request, xform): """ @@ -77,9 +83,9 @@ def _create_enketo_urls(request, xform): MetaData.enketo_single_submit_url(xform, single_url) data["single_url"] = single_url except ConnectionError as e: - logging.exception("Connection Error: %s" % e) + logging.exception("Connection Error: %s", e) except EnketoError as e: - logging.exception("Enketo Error: %s" % e.message) + logging.exception("Enketo Error: %s", e.message) return data @@ -93,17 +99,23 @@ def _set_cache(cache_key, cache_data, obj): :param obj: :return: Data that has been cached """ - cache.set("{}{}".format(cache_key, obj.pk), cache_data) + cache.set(f"{cache_key}{obj.pk}", cache_data) return cache_data def user_to_username(item): + """ + Replaces the item["user"] user object with the to user.username. + """ item["user"] = item["user"].username return item def clean_public_key(value): + """ + Strips public key comments and spaces from a public key ``value``. + """ if value.startswith("-----BEGIN PUBLIC KEY-----") and value.endswith( "-----END PUBLIC KEY-----" ): @@ -114,6 +126,8 @@ def clean_public_key(value): .rstrip() ) + return value + class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): """ @@ -126,9 +140,13 @@ class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): def __init__(self, *args, **kwargs): self.lookup_fields = kwargs.pop("lookup_fields", self.lookup_fields) - super(MultiLookupIdentityField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + # pylint: disable=redefined-builtin def get_url(self, obj, view_name, request, format): + """ + Returns URL to the given object. + """ kwargs = {} for model_field, url_param in self.lookup_fields: attr = obj @@ -136,16 +154,21 @@ def get_url(self, obj, view_name, request, format): attr = getattr(attr, field) kwargs[url_param] = attr - if not format and hasattr(self, "format"): - fmt = self.format - else: - fmt = format + fmt = self.format if not format and hasattr(self, "format") else format return reverse(view_name, kwargs=kwargs, request=request, format=fmt) -class XFormMixin(object): +class XFormMixin: + """ + XForm mixins + """ + + # pylint: disable=no-self-use def get_xls_available(self, obj): + """ + Returns True if ``obj.xls.url`` is not None, indicates XLS is present. + """ available = False if obj and obj.xls: try: @@ -159,17 +182,21 @@ def _get_metadata(self, obj, key): for m in obj.metadata_set.all(): if m.data_type == key: return m.data_value - else: - return obj.metadata_set.all() + return obj.metadata_set.all() + + # pylint: disable=no-self-use def get_users(self, obj): + """ + Returns a list of users based on XForm permissions. + """ xform_perms = [] if obj: - xform_perms = cache.get("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk)) + xform_perms = cache.get(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}") if xform_perms: return xform_perms - cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) + cache.set(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}", xform_perms) data = {} for perm in obj.xformuserobjectpermission_set.all(): if perm.user_id not in data: @@ -193,13 +220,16 @@ def get_users(self, obj): xform_perms = list(itervalues(data)) - cache.set("{}{}".format(XFORM_PERMISSIONS_CACHE, obj.pk), xform_perms) + cache.set(f"{XFORM_PERMISSIONS_CACHE}{obj.pk}", xform_perms) return xform_perms def get_enketo_url(self, obj): + """ + Returns Enketo URL for given ``obj``. + """ if obj: - _enketo_url = cache.get("{}{}".format(ENKETO_URL_CACHE, obj.pk)) + _enketo_url = cache.get(f"{ENKETO_URL_CACHE}{obj.pk}") if _enketo_url: return _enketo_url @@ -213,9 +243,12 @@ def get_enketo_url(self, obj): return None def get_enketo_single_submit_url(self, obj): + """ + Returns single submit Enketo URL for given ``obj``. + """ if obj: _enketo_single_submit_url = cache.get( - "{}{}".format(ENKETO_SINGLE_SUBMIT_URL_CACHE, obj.pk) + f"{ENKETO_SINGLE_SUBMIT_URL_CACHE}{obj.pk}" ) if _enketo_single_submit_url: return _enketo_single_submit_url @@ -230,19 +263,21 @@ def get_enketo_single_submit_url(self, obj): return None def get_enketo_preview_url(self, obj): + """ + Returns preview Enketo URL for given ``obj``. + """ if obj: - _enketo_preview_url = cache.get( - "{}{}".format(ENKETO_PREVIEW_URL_CACHE, obj.pk) - ) + _enketo_preview_url = cache.get(f"{ENKETO_PREVIEW_URL_CACHE}{obj.pk}") if _enketo_preview_url: return _enketo_preview_url url = self._get_metadata(obj, "enketo_preview_url") if url is None: - try: - enketo_urls = _create_enketo_urls(self.context.get("request"), obj) - url = enketo_urls.get("preview_url") - except Exception: + enketo_urls = _create_enketo_urls(self.context.get("request"), obj) + url = ( + enketo_urls["preview_url"] if "preview_url" in enketo_urls else url + ) + if url is None: return url return _set_cache(ENKETO_PREVIEW_URL_CACHE, url, obj) @@ -251,7 +286,7 @@ def get_enketo_preview_url(self, obj): def get_data_views(self, obj): if obj: - key = "{}{}".format(XFORM_LINKED_DATAVIEWS, obj.pk) + key = f"{XFORM_LINKED_DATAVIEWS}{obj.pk}" data_views = cache.get(key) if data_views: return data_views @@ -267,18 +302,22 @@ def get_data_views(self, obj): return data_views return [] + # pylint: disable=no-self-use def get_num_of_submissions(self, obj): + """ + Returns number of submissions. + """ if obj: - key = "{}{}".format(XFORM_COUNT, obj.pk) + key = f"{XFORM_COUNT}{obj.pk}" count = cache.get(key) if count: return count - force_update = True if obj.is_merged_dataset else False - count = obj.submission_count(force_update) + count = obj.submission_count(obj.is_merged_dataset) cache.set(key, count) return count + return 0 def get_last_submission_time(self, obj): """Return datetime of last submission @@ -427,10 +466,14 @@ class Meta: "deleted_by", ) + # pylint: disable=no-self-use def get_metadata(self, obj): + """ + Returns XForn ``obj`` metadata. + """ xform_metadata = [] if obj: - xform_metadata = cache.get("{}{}".format(XFORM_METADATA_CACHE, obj.pk)) + xform_metadata = cache.get(f"{XFORM_METADATA_CACHE}{obj.pk}") if xform_metadata: return xform_metadata @@ -439,7 +482,7 @@ def get_metadata(self, obj): obj.metadata_set.all(), many=True, context=self.context ).data ) - cache.set("{}{}".format(XFORM_METADATA_CACHE, obj.pk), xform_metadata) + cache.set(f"{XFORM_METADATA_CACHE}{obj.pk}", xform_metadata) return xform_metadata @@ -451,10 +494,10 @@ def validate_public_key(self, value): # pylint: disable=no-self-use """ try: load_pem_public_key(value.encode("utf-8"), backend=default_backend()) - except ValueError: + except ValueError as e: raise serializers.ValidationError( _("The public key is not a valid base64 RSA key") - ) + ) from e return clean_public_key(value) def _check_if_allowed_public(self, value): # pylint: disable=no-self-use @@ -478,14 +521,18 @@ def validate_public(self, value): """ return self._check_if_allowed_public(value) + # pylint: disable=no-self-use def get_form_versions(self, obj): + """ + Returns all form versions. + """ versions = [] if obj: - versions = cache.get("{}{}".format(XFORM_DATA_VERSIONS, obj.pk)) + versions = cache.get(f"{XFORM_DATA_VERSIONS}{obj.pk}") if versions: return versions - elif obj.num_of_submissions > SUBMISSION_RETRIEVAL_THRESHOLD: + if obj.num_of_submissions > SUBMISSION_RETRIEVAL_THRESHOLD: return [] versions = list( @@ -495,29 +542,44 @@ def get_form_versions(self, obj): ) if versions: - cache.set("{}{}".format(XFORM_DATA_VERSIONS, obj.pk), list(versions)) + cache.set(f"{XFORM_DATA_VERSIONS}{obj.pk}", list(versions)) return versions class XFormCreateSerializer(XFormSerializer): + """ + XForm serializer that is only relevant during the XForm publishing process. + """ + has_id_string_changed = serializers.SerializerMethodField() + # pylint: disable=no-self-use def get_has_id_string_changed(self, obj): + """ + Returns the value of ``obj.has_id_string_changed`` + """ return obj.has_id_string_changed class XFormListSerializer(serializers.Serializer): - formID = serializers.ReadOnlyField(source="id_string") + """ + XForm serializer for OpenRosa form list API. + """ + + formID = serializers.ReadOnlyField(source="id_string") # noqa name = serializers.ReadOnlyField(source="title") version = serializers.ReadOnlyField() hash = serializers.ReadOnlyField() - descriptionText = serializers.ReadOnlyField(source="description") - downloadUrl = serializers.SerializerMethodField("get_url") - manifestUrl = serializers.SerializerMethodField("get_manifest_url") + descriptionText = serializers.ReadOnlyField(source="description") # noqa + downloadUrl = serializers.SerializerMethodField("get_url") # noqa + manifestUrl = serializers.SerializerMethodField("get_manifest_url") # noqa @check_obj def get_url(self, obj): + """ + Returns XForm download URL. + """ kwargs = {"pk": obj.pk, "username": obj.user.username} request = self.context.get("request") @@ -525,6 +587,9 @@ def get_url(self, obj): @check_obj def get_manifest_url(self, obj): + """ + Return manifest URL. + """ kwargs = {"pk": obj.pk, "username": obj.user.username} request = self.context.get("request") object_list = MetaData.objects.filter(data_type="media", object_id=obj.pk) @@ -536,10 +601,13 @@ def get_manifest_url(self, obj): class XFormManifestSerializer(serializers.Serializer): filename = serializers.SerializerMethodField() hash = serializers.SerializerMethodField() - downloadUrl = serializers.SerializerMethodField("get_url") + downloadUrl = serializers.SerializerMethodField("get_url") # noqa @check_obj def get_url(self, obj): + """ + Return media download URL. + """ kwargs = { "pk": obj.content_object.pk, "username": obj.content_object.user.username, @@ -555,17 +623,19 @@ def get_url(self, obj): group_delimiter = self.context.get(GROUP_DELIMETER_TAG) repeat_index_tags = self.context.get(REPEAT_INDEX_TAGS) if group_delimiter and repeat_index_tags and fmt == "csv": - return url + "?%s=%s&%s=%s" % ( - GROUP_DELIMETER_TAG, - group_delimiter, - REPEAT_INDEX_TAGS, - repeat_index_tags, + return url + ( + f"?{GROUP_DELIMETER_TAG}={group_delimiter}" + f"&{REPEAT_INDEX_TAGS}={repeat_index_tags}" ) return url + # pylint: disable=no-self-use @check_obj def get_hash(self, obj): + """ + Returns MD5 hash based on last_submission_time for a media linked form. + """ filename = obj.data_value hsh = obj.file_hash parts = filename.split(" ") @@ -588,22 +658,25 @@ def get_hash(self, obj): xform = data_view.xform if xform and xform.last_submission_time: - hsh = "md5:%s" % ( - md5( - xform.last_submission_time.isoformat().encode("utf-8") - ).hexdigest() - ) + md5_hash = md5( + xform.last_submission_time.isoformat().encode("utf-8") + ).hexdigest() + hsh = f"md5:{md5_hash}" - return "%s" % (hsh or "md5:") + return f"{hsh or 'md5:'}" + # pylint: disable=no-self-use @check_obj def get_filename(self, obj): + """ + Returns media filename. + """ filename = obj.data_value parts = filename.split(" ") # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = "%s.csv" % parts[2] + filename = "{parts[2]}.csv" else: try: URLValidator()(filename) @@ -617,6 +690,10 @@ def get_filename(self, obj): class XFormVersionListSerializer(serializers.ModelSerializer): + """ + XFormVersion list API serializer + """ + xform = serializers.HyperlinkedRelatedField( view_name="xform-detail", lookup_field="pk", diff --git a/onadata/libs/utils/email.py b/onadata/libs/utils/email.py index 9b11a5ada7..d725941b6f 100644 --- a/onadata/libs/utils/email.py +++ b/onadata/libs/utils/email.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +email utility functions. +""" from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -6,17 +10,22 @@ def get_verification_url(redirect_url, request, verification_key): + """Returns the verification_url""" verification_url = getattr(settings, "VERIFICATION_URL", None) url = verification_url or reverse("userprofile-verify-email", request=request) query_params_dict = {"verification_key": verification_key} - redirect_url and query_params_dict.update({"redirect_url": redirect_url}) + if redirect_url: + query_params_dict.update({"redirect_url": redirect_url}) query_params_string = urlencode(query_params_dict) - verification_url = "{}?{}".format(url, query_params_string) + verification_url = f"{url}?{query_params_string}" return verification_url def get_verification_email_data(email, username, verification_url, request): + """ + Returns the verification email content + """ email_data = {"email": email} ctx_dict = { @@ -37,7 +46,7 @@ def get_verification_email_data(email, username, verification_url, request): return email_data -def get_account_lockout_email_data(username, ip, end=False): +def get_account_lockout_email_data(username, ip_address, end=False): """Generates both the email upon start and end of account lockout""" message_path = "account_lockout/lockout_start.txt" subject_path = "account_lockout/lockout_email_subject.txt" @@ -45,7 +54,7 @@ def get_account_lockout_email_data(username, ip, end=False): message_path = "account_lockout/lockout_end.txt" ctx_dict = { "username": username, - "remote_ip": ip, + "remote_ip": ip_address, "lockout_time": getattr(settings, "LOCKOUT_TIME", 1800) / 60, "support_email": getattr(settings, "SUPPORT_EMAIL", "support@example.com"), } From 4bf0ac43bb37cdb33c099e713e3fe629ad97fd8b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 1 May 2022 14:19:58 +0300 Subject: [PATCH 079/234] batch: cleanup --- .../commands/fix_readonly_role_perms.py | 69 ++++++++++-------- onadata/libs/authentication.py | 5 +- .../serializers/password_reset_serializer.py | 71 ++++++++++++++++--- onadata/libs/serializers/widget_serializer.py | 20 ++++++ onadata/libs/serializers/xform_serializer.py | 2 +- 5 files changed, 125 insertions(+), 42 deletions(-) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 268afe4fb1..265f459ca6 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +fix_readonly_role_perms - Reassign permission to the model when permissions are changed +""" from guardian.shortcuts import get_perms from django.core.management.base import BaseCommand, CommandError @@ -21,10 +25,41 @@ from onadata.libs.utils.model_tools import queryset_iterator +# pylint: disable=invalid-name User = get_user_model() +def check_role(role_class, user, obj, new_perm=None): + """ + Test if the user has the role for the object provided + :param role_class: + :param user: + :param obj: + :param new_perm: + :return: + """ + new_perm = new_perm if new_perm is None else [] + # remove the new permission because the old model doesnt have it + perm_list = role_class.class_to_permissions[type(obj)] + old_perm_set = set(perm_list) + newly_added_perm = set(new_perm) + + if newly_added_perm.issubset(old_perm_set): + diff_set = old_perm_set.difference(newly_added_perm) + + if isinstance(user, Team): + return set(get_perms(user, obj)) == diff_set + + return user.has_perms(list(diff_set), obj) + return False + + class Command(BaseCommand): + """ + fix_readonly_role_perms - Reassign permission to the model when + permissions are changed + """ + args = "" help = _("Reassign permission to the model when permissions are changed") @@ -105,38 +140,16 @@ def reassign_perms(self, user, app, model, new_perm): # For each role reassign the perms for role_class in reversed(ROLES): - # want to only process for readonly perms - if role_class.user_has_role(user, obj) or role_class not in [ + not_readonly = role_class.user_has_role( + user, obj + ) or role_class not in [ ReadOnlyRoleNoDownload, ReadOnlyRole, - ]: + ] + if not_readonly: continue - if self.check_role(role_class, user, obj, new_perm): + if check_role(role_class, user, obj, new_perm): # If true role_class.add(user, obj) break - - def check_role(self, role_class, user, obj, new_perm=None): - """ - Test if the user has the role for the object provided - :param role_class: - :param user: - :param obj: - :param new_perm: - :return: - """ - new_perm = new_perm if new_perm is None else [] - # remove the new permission because the old model doesnt have it - perm_list = role_class.class_to_permissions[type(obj)] - old_perm_set = set(perm_list) - newly_added_perm = set(new_perm) - - if newly_added_perm.issubset(old_perm_set): - diff_set = old_perm_set.difference(newly_added_perm) - - if isinstance(user, Team): - return set(get_perms(user, obj)) == diff_set - - return user.has_perms(list(diff_set), obj) - return False diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index 1f14520422..97ab0f386c 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -"""Authentication classes. +""" +Authentication classes. """ from __future__ import unicode_literals @@ -92,7 +93,7 @@ def get_api_token(cookie_jwt): except BadSignature as e: raise exceptions.AuthenticationFailed(_(f"Bad Signature: {e}")) from e except jwt.DecodeError as e: - raise exceptions.AuthenticationFailed(_("JWT DecodeError: {e}")) from e + raise exceptions.AuthenticationFailed(_(f"JWT DecodeError: {e}")) from e except Token.DoesNotExist as e: raise exceptions.AuthenticationFailed(_("Invalid token")) from e diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index 1eaeb47fd4..6f2a01b929 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -1,7 +1,10 @@ -from builtins import bytes as b +# -*- coding: utf-8 -*- +""" +Password reset serializer. +""" from six.moves.urllib.parse import urlparse -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.core.mail import send_mail from django.template import loader @@ -14,8 +17,14 @@ from onadata.libs.utils.user_auth import invalidate_and_regen_tokens +# pylint: disable=invalid-name +User = get_user_model() + + class CustomPasswordResetTokenGenerator(PasswordResetTokenGenerator): - """Custom Password Token Generator Class.""" + """ + Custom Password Token Generator Class. + """ def _make_hash_value(self, user, timestamp): # Include user email alongside user password to the generated token @@ -38,6 +47,7 @@ def _make_hash_value(self, user, timestamp): default_token_generator = CustomPasswordResetTokenGenerator() +# pylint: disable=unused-argument,too-many-arguments def get_password_reset_email( user, reset_url, @@ -49,7 +59,7 @@ def get_password_reset_email( """Creates the subject and email body for password reset email.""" result = urlparse(reset_url) site_name = domain = result.hostname - encoded_username = urlsafe_base64_encode(b(user.username.encode("utf-8"))) + encoded_username = urlsafe_base64_encode(bytes(user.username.encode("utf-8"))) c = { "email": user.email, "domain": domain, @@ -71,24 +81,37 @@ def get_password_reset_email( def get_user_from_uid(uid): + """ + Return user from base64 encoded ``uid``. + """ if uid is None: raise serializers.ValidationError(_("uid is required!")) try: uid = urlsafe_base64_decode(uid) user = User.objects.get(pk=uid) - except (TypeError, ValueError, OverflowError, User.DoesNotExist): - raise serializers.ValidationError(_("Invalid uid %s") % uid) + except (TypeError, ValueError, OverflowError, User.DoesNotExist) as e: + raise serializers.ValidationError(_("Invalid uid {uid}")) from e return user -class PasswordResetChange(object): +# pylint: disable=too-few-public-methods +class PasswordResetChange: + """ + Class resets and changes the password. + + Class imitates a model functionality for use with PasswordResetSerializer + """ + def __init__(self, uid, new_password, token): self.uid = uid self.new_password = new_password self.token = token def save(self): + """ + Set a new user password and invalidate/regenerate tokens. + """ user = get_user_from_uid(self.uid) if user: user.set_password(self.new_password) @@ -96,12 +119,20 @@ def save(self): invalidate_and_regen_tokens(user) -class PasswordReset(object): +# pylint: disable=too-few-public-methods +class PasswordReset: + """ + Class resets the password and sends the reset email. + + Class imitates a model functionality for use with PasswordResetSerializer + """ + def __init__(self, email, reset_url, email_subject=None): self.email = email self.reset_url = reset_url self.email_subject = email_subject + # pylint: disable=unused-argument def save( self, subject_template_name="registration/password_reset_subject.txt", @@ -133,24 +164,34 @@ def save( send_mail(subject, email, from_email, [user.email]) +# pylint: disable=abstract-method class PasswordResetSerializer(serializers.Serializer): + """ + Password reset serializer. + """ + email = serializers.EmailField(label=_("Email"), max_length=254) reset_url = serializers.URLField(label=_("Reset URL"), max_length=254) email_subject = serializers.CharField( label=_("Email Subject"), required=False, max_length=78, allow_blank=True ) + # pylint: disable=no-self-use def validate_email(self, value): + """ + Validates the email. + """ users = User.objects.filter(email__iexact=value) if users.count() == 0: - raise serializers.ValidationError( - _("User '%(value)s' does not exist." % {"value": value}) - ) + raise serializers.ValidationError(_(f"User '{value}' does not exist.")) return value def validate_email_subject(self, value): + """ + Validate the email subject is not empty. + """ if len(value) == 0: return None @@ -164,11 +205,19 @@ def create(self, validated_data): class PasswordResetChangeSerializer(serializers.Serializer): + """ + Reset and change password serializer. + """ + uid = serializers.CharField(max_length=50) new_password = serializers.CharField(min_length=4, max_length=128) token = serializers.CharField(max_length=128) + # pylint: disable=no-self-use def validate_uid(self, value): + """ + Validate the user uid. + """ get_user_from_uid(value) return value diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index f5a8ef1cd5..e82d48206a 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -22,6 +22,10 @@ class GenericRelatedField(serializers.HyperlinkedRelatedField): + """ + GenericRelatedField - handle related field relations for XForm and DataView + """ + default_error_messages = { "incorrect_match": _("`{input}` is not a valid relation.") } @@ -35,18 +39,23 @@ def __init__(self, *args, **kwargs): super(serializers.RelatedField, self).__init__(*args, **kwargs) def _setup_field(self, view_name): + # pylint: disable=attribute-defined-outside-init self.lookup_url_kwarg = self.lookup_field if view_name == "xform-detail": + # pylint: disable=attribute-defined-outside-init self.queryset = XForm.objects.all() if view_name == "dataviews-detail": + # pylint: disable=attribute-defined-outside-init self.queryset = DataView.objects.all() def to_representation(self, value): if isinstance(value, XForm): + # pylint: disable=attribute-defined-outside-init self.view_name = "xform-detail" elif isinstance(value, DataView): + # pylint: disable=attribute-defined-outside-init self.view_name = "dataviews-detail" else: raise Exception(_("Unknown type for content_object.")) @@ -103,7 +112,12 @@ class WidgetSerializer(serializers.HyperlinkedModelSerializer): order = serializers.IntegerField(required=False) metadata = JsonField(required=False) + # pylint: disable=too-few-public-methods class Meta: + """ + Meta model - specifies the fields in the Model Widget for the serializer + """ + model = Widget fields = ( "id", @@ -123,6 +137,9 @@ class Meta: ) def get_data(self, obj): + """ + Return the Widget.query_data(obj) + """ # Get the request obj request = self.context.get("request") @@ -167,6 +184,9 @@ def validate(self, attrs): return attrs def validate_content_object(self, value): + """ + Validate if a user is the owner f the organization. + """ request = self.context.get("request") users = get_users_with_perms( value.project, attach_perms=False, with_group_users=False diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 5824806a49..30e1e3d597 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -676,7 +676,7 @@ def get_filename(self, obj): # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = "{parts[2]}.csv" + filename = f"{parts[2]}.csv" else: try: URLValidator()(filename) From 08a2ff5e4cf8460731f441fe024069a3cabab6ae Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 1 May 2022 16:02:18 +0300 Subject: [PATCH 080/234] batch: cleanup --- onadata/apps/logger/models/data_view.py | 30 ++- onadata/apps/logger/models/osmdata.py | 41 ++-- onadata/apps/logger/xform_instance_parser.py | 194 ++++++++++++------- 3 files changed, 163 insertions(+), 102 deletions(-) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index b5a41efdff..279c0898ed 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -4,7 +4,6 @@ """ import datetime import json -from builtins import str as text from django.conf import settings from django.contrib.gis.db import models @@ -90,8 +89,8 @@ def has_attachments_fields(data_view): attachments += get_elements_of_type(xform, element_type) if attachments: - for a in data_view.columns: - if a in attachments: + for attachment in data_view.columns: + if attachment in attachments: return True return False @@ -192,7 +191,7 @@ def has_instance(self, instance): ) params = [self.xform.pk, instance.id] + where_params - cursor.execute(sql, [text(i) for i in params]) + cursor.execute(sql, [str(i) for i in params]) records = None for row in cursor.fetchall(): records = row[0] @@ -232,16 +231,16 @@ def _get_where_clause( # pylint: disable=too-many-locals where = [] where_params = [] - query = data_view.query + queries = data_view.query or_where = [] or_params = [] - for qu in query: - comp = qu.get("filter") - column = qu.get("column") - value = qu.get("value") - condi = qu.get("condition") + for query in queries: + comp = query.get("filter") + column = query.get("column") + value = query.get("value") + condi = query.get("condition") json_str = _json_sql_str( column, known_integers, known_dates, known_decimals @@ -252,10 +251,10 @@ def _get_where_clause( # pylint: disable=too-many-locals if condi and condi.lower() == "or": or_where = append_where_list(comp, or_where, json_str) - or_params.extend((column, text(value))) + or_params.extend((column, str(value))) else: where = append_where_list(comp, where, json_str) - where_params.extend((column, text(value))) + where_params.extend((column, str(value))) if or_where: or_where = ["".join(["(", " OR ".join(or_where), ")"])] @@ -276,7 +275,7 @@ def parse_json(data): params = [] if params is None else params cursor = connection.cursor() - sql_params = tuple(i if isinstance(i, tuple) else text(i) for i in params) + sql_params = tuple(i if isinstance(i, tuple) else str(i) for i in params) if count: from_pos = sql.upper().find(" FROM") @@ -416,10 +415,7 @@ def query_data( # pylint: disable=too-many-arguments filter_query, ) - try: - records = list(DataView.query_iterator(sql, columns, params, count)) - except Exception as e: - return {"error": _(text(e))} + records = list(DataView.query_iterator(sql, columns, params, count)) return records diff --git a/onadata/apps/logger/models/osmdata.py b/onadata/apps/logger/models/osmdata.py index 352bc496bf..d79372fdaf 100644 --- a/onadata/apps/logger/models/osmdata.py +++ b/onadata/apps/logger/models/osmdata.py @@ -10,52 +10,57 @@ class OsmData(models.Model): """ OSM Data information from a submission instance """ + instance = models.ForeignKey( - 'logger.Instance', related_name='osm_data', on_delete=models.CASCADE) + "logger.Instance", related_name="osm_data", on_delete=models.CASCADE + ) xml = models.TextField() osm_id = models.CharField(max_length=20) - osm_type = models.CharField(max_length=10, default='way') + osm_type = models.CharField(max_length=10, default="way") tags = JSONField(default=dict, null=False) geom = models.GeometryCollectionField() filename = models.CharField(max_length=255) - field_name = models.CharField(max_length=255, blank=True, default='') + field_name = models.CharField(max_length=255, blank=True, default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) class Meta: - app_label = 'logger' - unique_together = ('instance', 'field_name') + app_label = "logger" + unique_together = ("instance", "field_name") @classmethod def get_tag_keys(cls, xform, field_path, include_prefix=False): + """ + Returns sorted tag keys. + """ query = OsmData.objects.raw( - '''SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata" INNER JOIN "logger_instance" ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" ) WHERE "logger_instance"."xform_id" = %s AND field_name = %s''', # noqa - [xform.pk, field_path] + """SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata" INNER JOIN "logger_instance" ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" ) WHERE "logger_instance"."xform_id" = %s AND field_name = %s""", # noqa + [xform.pk, field_path], ) - prefix = field_path + u':' if include_prefix else u'' + prefix = field_path + ":" if include_prefix else "" return sorted([prefix + key.id for key in query]) def get_tags_with_prefix(self): - doc = { - self.field_name + ':' + self.osm_type + ':id': self.osm_id - } + doc = {self.field_name + ":" + self.osm_type + ":id": self.osm_id} for k, v in self.tags.items(): - doc[self.field_name + ':' + k] = v + doc[self.field_name + ":" + k] = v return doc def _set_centroid_in_tags(self): self.tags = self.tags if isinstance(self.tags, dict) else {} if self.geom is not None: - # pylint: disable=E1101 - self.tags.update({ - "ctr:lon": self.geom.centroid.x, - "ctr:lat": self.geom.centroid.y, - }) + # pylint: disable=no-member + self.tags.update( + { + "ctr:lon": self.geom.centroid.x, + "ctr:lat": self.geom.centroid.y, + } + ) def save(self, *args, **kwargs): self._set_centroid_in_tags() - super(OsmData, self).save(*args, **kwargs) + super().save(*args, **kwargs) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index 7d23840bbf..c84594ad6e 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -1,9 +1,13 @@ +# -*- coding: utf-8 -*- +""" +XForm submission XML parser utility functions. +""" import logging import re -import dateutil.parser -from builtins import str as text from xml.dom import minidom, Node +import dateutil.parser + from django.utils.encoding import smart_text, smart_str from django.utils.translation import ugettext as _ @@ -55,6 +59,9 @@ class NonUniqueFormIdError(Exception): def get_meta_from_xml(xml_str, meta_name): + """ + Return the meta section of an XForm submission XML. + """ xml = clean_and_parse_xml(xml_str) children = xml.childNodes # children ideally contains a single element @@ -79,17 +86,22 @@ def get_meta_from_xml(xml_str, meta_name): if n.nodeType == Node.ELEMENT_NODE and ( n.tagName.lower() == meta_name.lower() - or n.tagName.lower() == "orx:%s" % meta_name.lower() + or n.tagName.lower() == f"orx:{meta_name.lower()}" ) ] if len(uuid_tags) == 0: return None uuid_tag = uuid_tags[0] + return uuid_tag.firstChild.nodeValue.strip() if uuid_tag.firstChild else None def get_uuid_from_xml(xml): + """ + Returns the uuid of an XForm submisison XML + """ + def _uuid_only(uuid, regex): matches = regex.match(uuid) if matches and len(matches.groups()) > 0: @@ -100,21 +112,28 @@ def _uuid_only(uuid, regex): regex = re.compile(r"uuid:(.*)") if uuid: return _uuid_only(uuid, regex) + # check in survey_node attributes xml = clean_and_parse_xml(xml) children = xml.childNodes + # children ideally contains a single element # that is the parent of all survey elements if children.length == 0: raise ValueError(_("XML string must have a survey element.")) + survey_node = children[0] uuid = survey_node.getAttribute("instanceID") if uuid != "": return _uuid_only(uuid, regex) + return None def get_submission_date_from_xml(xml): + """ + Returns submissionDate from an XML submission. + """ # check in survey_node attributes xml = clean_and_parse_xml(xml) children = xml.childNodes @@ -123,13 +142,16 @@ def get_submission_date_from_xml(xml): if children.length == 0: raise ValueError(_("XML string must have a survey element.")) survey_node = children[0] - submissionDate = survey_node.getAttribute("submissionDate") - if submissionDate != "": - return dateutil.parser.parse(submissionDate) + submission_date = survey_node.getAttribute("submissionDate") + if submission_date != "": + return dateutil.parser.parse(submission_date) return None def get_deprecated_uuid_from_xml(xml): + """ + Returns the deprecatedID from submission XML + """ uuid = get_meta_from_xml(xml, "deprecatedID") regex = re.compile(r"uuid:(.*)") if uuid: @@ -140,76 +162,82 @@ def get_deprecated_uuid_from_xml(xml): def clean_and_parse_xml(xml_string): + """ + Removes spaces between XML tags in ``xml_string`` + + Returns an XML object via minidom.parseString(xml_string) + """ clean_xml_str = xml_string.strip() clean_xml_str = re.sub(r">\s+<", "><", smart_text(clean_xml_str)) xml_obj = minidom.parseString(smart_str(clean_xml_str)) return xml_obj -def _xml_node_to_dict(node, repeats=[], encrypted=False): +# pylint: disable=too-many-branches +def _xml_node_to_dict(node, repeats=None, encrypted=False): + repeats = [] if repeats is None else repeats if len(node.childNodes) == 0: # there's no data for this leaf node return None - elif len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: + if len(node.childNodes) == 1 and node.childNodes[0].nodeType == node.TEXT_NODE: # there is data for this leaf node return {node.nodeName: node.childNodes[0].nodeValue} - else: - # this is an internal node - value = {} - - for child in node.childNodes: - # handle CDATA text section - if child.nodeType == child.CDATA_SECTION_NODE: - return {child.parentNode.nodeName: child.nodeValue} - - d = _xml_node_to_dict(child, repeats) - - if d is None: - continue - - child_name = child.nodeName - child_xpath = xpath_from_xml_node(child) - if list(d) != [child_name]: - raise AssertionError() - node_type = dict - # check if name is in list of repeats and make it a list if so - # All the photo attachments in an encrypted form use name media - if child_xpath in repeats or (encrypted and child_name == "media"): - node_type = list - - if node_type == dict: - if child_name not in value: - value[child_name] = d[child_name] - else: - # node is repeated, aggregate node values - node_value = value[child_name] - # 1. check if the node values is a list - if not isinstance(node_value, list): - # if not a list create - value[child_name] = [node_value] - # 2. parse the node - d = _xml_node_to_dict(child, repeats) - # 3. aggregate - value[child_name].append(d[child_name]) + # this is an internal node + value = {} + + for child in node.childNodes: + # handle CDATA str section + if child.nodeType == child.CDATA_SECTION_NODE: + return {child.parentNode.nodeName: child.nodeValue} + + child_dict = _xml_node_to_dict(child, repeats) + + if child_dict is None: + continue + + child_name = child.nodeName + child_xpath = xpath_from_xml_node(child) + if list(child_dict) != [child_name]: + raise AssertionError() + node_type = dict + # check if name is in list of repeats and make it a list if so + # All the photo attachments in an encrypted form use name media + if child_xpath in repeats or (encrypted and child_name == "media"): + node_type = list + + if node_type == dict: + if child_name not in value: + value[child_name] = child_dict[child_name] else: - if child_name not in value: - value[child_name] = [d[child_name]] - else: - value[child_name].append(d[child_name]) - if value == {}: - return None + # node is repeated, aggregate node values + node_value = value[child_name] + # 1. check if the node values is a list + if not isinstance(node_value, list): + # if not a list create + value[child_name] = [node_value] + # 2. parse the node + child_dict = _xml_node_to_dict(child, repeats) + # 3. aggregate + value[child_name].append(child_dict[child_name]) else: - return {node.nodeName: value} + if child_name not in value: + value[child_name] = [child_dict[child_name]] + else: + value[child_name].append(child_dict[child_name]) + if not value: + return None + return {node.nodeName: value} -def _flatten_dict(d, prefix): + +def _flatten_dict(data_dict, prefix): """ Return a list of XPath, value pairs. - :param d: A dictionary + :param data_dict: A python dictionary object :param prefix: A list of prefixes """ - for key, value in d.items(): + for key, value in data_dict.items(): new_prefix = prefix + [key] if isinstance(value, dict): @@ -226,7 +254,7 @@ def _flatten_dict(d, prefix): # hack: removing [1] index to be consistent across # surveys that have a single repitition of the # loop versus mutliple. - item_prefix[-1] += "[%s]" % text(i + 1) + item_prefix[-1] += f"[{str(i + 1)}]" if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): @@ -237,14 +265,15 @@ def _flatten_dict(d, prefix): yield (new_prefix, value) -def _flatten_dict_nest_repeats(d, prefix): +def _flatten_dict_nest_repeats(data_dict, prefix): """ Return a list of XPath, value pairs. - :param d: A dictionary + :param data_dict: A python dictionary object + :param prefix: A list of prefixes :param prefix: A list of prefixes """ - for key, value in d.items(): + for key, value in data_dict.items(): new_prefix = prefix + [key] if isinstance(value, dict): for pair in _flatten_dict_nest_repeats(value, new_prefix): @@ -252,14 +281,14 @@ def _flatten_dict_nest_repeats(d, prefix): elif isinstance(value, list): repeats = [] - for i, item in enumerate(value): + for _i, item in enumerate(value): item_prefix = list(new_prefix) # make a copy if isinstance(item, dict): repeat = {} - for path, value in _flatten_dict_nest_repeats(item, item_prefix): - # TODO: this only considers the first level of repeats - repeat.update({"/".join(path[1:]): value}) + for path, r_value in _flatten_dict_nest_repeats(item, item_prefix): + # This only considers the first level of repeats + repeat.update({"/".join(path[1:]): r_value}) repeats.append(repeat) else: repeats.append({"/".join(item_prefix[1:]): item}) @@ -281,6 +310,9 @@ def _gather_parent_node_list(node): def xpath_from_xml_node(node): + """ + Returns the xpath of an XML node. + """ node_names = _gather_parent_node_list(node) return "/".join(node_names[1:]) @@ -299,12 +331,21 @@ def _get_all_attributes(node): yield pair -class XFormInstanceParser(object): +class XFormInstanceParser: + """ + XFormInstanceParser - parses an XML string into an XML object. + """ + def __init__(self, xml_str, data_dictionary): + # pylint: disable=invalid-name self.dd = data_dictionary self.parse(xml_str) + self._attributes = {} def parse(self, xml_str): + """ + Parses a submission XML into a python dictionary object. + """ self._xml_obj = clean_and_parse_xml(xml_str) self._root_node = self._xml_obj.documentElement repeats = [ @@ -349,19 +390,28 @@ def _set_attributes(self): if key in self._attributes: logger = logging.getLogger("console_logger") logger.debug( - "Skipping duplicate attribute: %s" " with value %s" % (key, value) + "Skipping duplicate attribute: %s" " with value %s", key, value ) - logger.debug(text(all_attributes)) + logger.debug(str(all_attributes)) else: self._attributes[key] = value def get_xform_id_string(self): + """ + Returns the submission XML `id` attribute. + """ return self._attributes["id"] def get_version(self): + """ + Returns the submission XML version attribute. + """ return self._attributes.get("version") def get_flat_dict_with_attributes(self): + """ + Adds the submission XML top level attributes to the resulting python object. + """ result = self.to_flat_dict().copy() result[XFORM_ID_STRING] = self.get_xform_id_string() @@ -373,15 +423,25 @@ def get_flat_dict_with_attributes(self): def xform_instance_to_dict(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a python object. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.to_dict() def xform_instance_to_flat_dict(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a flattened python object. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.to_flat_dict() def parse_xform_instance(xml_str, data_dictionary): + """ + Parses an XForm submission XML into a flattened python object + with additional attributes. + """ parser = XFormInstanceParser(xml_str, data_dictionary) return parser.get_flat_dict_with_attributes() From b65f088c39b0c632ea29f24d271113df90ac776b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 1 May 2022 17:19:59 +0300 Subject: [PATCH 081/234] Fix: handle KeyError exception --- onadata/apps/logger/xform_instance_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index c84594ad6e..e5f423ba1a 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -340,7 +340,6 @@ def __init__(self, xml_str, data_dictionary): # pylint: disable=invalid-name self.dd = data_dictionary self.parse(xml_str) - self._attributes = {} def parse(self, xml_str): """ @@ -382,6 +381,7 @@ def get_attributes(self): return self._attributes def _set_attributes(self): + # pylint: disable=attribute-defined-outside-init self._attributes = {} all_attributes = list(_get_all_attributes(self._root_node)) for key, value in all_attributes: From 2ce81f7ccf4f0247798fdb0405393db40a89176d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 1 May 2022 21:42:23 +0300 Subject: [PATCH 082/234] batch: cleanup --- onadata/apps/main/models/user_profile.py | 39 +- onadata/apps/viewer/models/parsed_instance.py | 354 +++++++++++------- onadata/apps/viewer/parsed_instance_tools.py | 23 +- onadata/libs/serializers/xform_serializer.py | 2 +- 4 files changed, 270 insertions(+), 148 deletions(-) diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 0862552ad9..9a9a549716 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -4,9 +4,8 @@ """ import requests from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.db import models -from django.db.models import JSONField from django.db.models.signals import post_save, pre_save from django.utils.translation import ugettext_lazy from guardian.shortcuts import get_perms_for_model, assign_perm @@ -23,6 +22,9 @@ REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" +# pylint: disable=invalid-name +User = get_user_model() + class UserProfile(models.Model): """ @@ -49,22 +51,31 @@ class UserProfile(models.Model): User, null=True, blank=True, on_delete=models.SET_NULL ) num_of_submissions = models.IntegerField(default=0) - metadata = JSONField(default=dict, blank=True) + metadata = models.JSONField(default=dict, blank=True) date_modified = models.DateTimeField(auto_now=True) def __str__(self): - return "%s[%s]" % (self.name, self.user.username) + return f"{self.name}[{self.user.username}]" @property def gravatar(self): + """ + Returns Gravatar URL. + """ return get_gravatar_img_link(self.user) @property def gravatar_exists(self): + """ + Check if Gravatar URL exists. + """ return gravatar_exists(self.user) @property def twitter_clean(self): + """ + Remove the '@' from twitter name. + """ if self.twitter.startswith("@"): return self.twitter[1:] return self.twitter @@ -77,7 +88,7 @@ def save( if self.pk is None and hasattr(settings, REQUIRE_AUTHENTICATION): self.require_auth = getattr(settings, REQUIRE_AUTHENTICATION) - super(UserProfile, self).save(force_insert, force_update, using, update_fields) + super().save(force_insert, force_update, using, update_fields) class Meta: app_label = "main" @@ -88,12 +99,20 @@ class Meta: ) +# pylint: disable=unused-argument def create_auth_token(sender, instance=None, created=False, **kwargs): + """ + Creates an authentication Token. + """ if created: Token.objects.create(user=instance) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """ + Assign's permission to the user that created the profile. + """ if created: for perm in get_perms_for_model(UserProfile): assign_perm(perm.codename, instance.user, instance) @@ -102,15 +121,19 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): assign_perm(perm.codename, instance.created_by, instance) +# pylint: disable=unused-argument def set_kpi_formbuilder_permissions(sender, instance=None, created=False, **kwargs): + """ + Assign KPI permissions to allow the user to create forms using KPI formbuilder. + """ if created: kpi_formbuilder_url = ( hasattr(settings, "KPI_FORMBUILDER_URL") and settings.KPI_FORMBUILDER_URL ) if kpi_formbuilder_url: requests.post( - "%s/%s" % (kpi_formbuilder_url, "grant-default-model-level-perms"), - headers={"Authorization": "Token %s" % instance.user.auth_token}, + f"{kpi_formbuilder_url}/grant-default-model-level-perms", + headers={"Authorization": "Token {instance.user.auth_token}"}, ) @@ -135,12 +158,14 @@ def set_kpi_formbuilder_permissions(sender, instance=None, created=False, **kwar ) +# pylint: disable=too-few-public-methods class UserProfileUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" content_object = models.ForeignKey(UserProfile, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class UserProfileGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 6522527e5e..1bbf05f569 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -1,86 +1,117 @@ +# -*- coding: utf-8 -*- +""" +ParsedInstance model +""" import datetime import json import types -from builtins import str as text -import six -from dateutil import parser from django.conf import settings -from django.db import connection -from django.db import models +from django.db import connection, models from django.db.models.query import EmptyQuerySet from django.utils.translation import ugettext as _ -from onadata.apps.logger.models.instance import Instance -from onadata.apps.logger.models.instance import _get_attachments_from_instance +import six +from dateutil import parser + +from onadata.apps.logger.models.instance import Instance, _get_attachments_from_instance from onadata.apps.logger.models.note import Note from onadata.apps.logger.models.xform import _encode_for_mongo -from onadata.apps.viewer.parsed_instance_tools import (get_where_clause, - NONE_JSON_FIELDS) +from onadata.apps.viewer.parsed_instance_tools import NONE_JSON_FIELDS, get_where_clause from onadata.libs.models.sorting import ( - json_order_by, json_order_by_params, sort_from_mongo_sort_str) -from onadata.libs.utils.common_tags import ID, UUID, ATTACHMENTS, \ - GEOLOCATION, SUBMISSION_TIME, MONGO_STRFTIME, BAMBOO_DATASET_ID, \ - DELETEDAT, TAGS, NOTES, SUBMITTED_BY, VERSION, DURATION, EDITED, \ - MEDIA_COUNT, TOTAL_MEDIA, MEDIA_ALL_RECEIVED, XFORM_ID, REVIEW_STATUS, \ - REVIEW_COMMENT, DATE_MODIFIED, REVIEW_DATE + json_order_by, + json_order_by_params, + sort_from_mongo_sort_str, +) +from onadata.libs.utils.common_tags import ( + ATTACHMENTS, + BAMBOO_DATASET_ID, + DATE_MODIFIED, + DELETEDAT, + DURATION, + EDITED, + GEOLOCATION, + ID, + MEDIA_ALL_RECEIVED, + MEDIA_COUNT, + MONGO_STRFTIME, + NOTES, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + SUBMISSION_TIME, + SUBMITTED_BY, + TAGS, + TOTAL_MEDIA, + UUID, + VERSION, + XFORM_ID, +) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _is_invalid_for_mongo -DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S" class ParseError(Exception): - pass + """ + Raise when an exception happens when parsing the XForm XML submission. + """ def datetime_from_str(text): + """ + Parses a datetime from a string and returns the datetime object. + """ # Assumes text looks like 2011-01-01T09:50:06.966 if text is None: return None - dt = None try: - dt = parser.parse(text) - except Exception: + return parser.parse(text) + except (TypeError, ValueError): return None - return dt + return None -def dict_for_mongo(d): - for key, value in d.items(): +def dict_for_mongo(item): + """ + Validates the keys of a python object. + """ + for key, value in item.items(): if isinstance(value, list): - value = [dict_for_mongo(e) - if isinstance(e, dict) else e for e in value] + value = [dict_for_mongo(e) if isinstance(e, dict) else e for e in value] elif isinstance(value, dict): value = dict_for_mongo(value) - elif key == '_id': + elif key == "_id": try: - d[key] = int(value) + item[key] = int(value) except ValueError: # if it is not an int don't convert it pass if _is_invalid_for_mongo(key): - del d[key] - d[_encode_for_mongo(key)] = value - return d + del item[key] + item[_encode_for_mongo(key)] = value + return item def get_name_from_survey_element(element): + """ + Returns the abbreviated xpath of an element. + """ return element.get_abbreviated_xpath() def _parse_sort_fields(fields): for field in fields: - key = field.lstrip('-') - if field.startswith('-') and key in NONE_JSON_FIELDS.keys(): + key = field.lstrip("-") + if field.startswith("-") and key in NONE_JSON_FIELDS: field = NONE_JSON_FIELDS.get(key) - yield f'-{field}' + yield f"-{field}" else: yield NONE_JSON_FIELDS.get(field, field) -def _query_iterator(sql, fields=None, params=[], count=False): - +def _query_iterator(sql, fields=None, params=None, count=False): def parse_json(data): try: return json.loads(data) @@ -88,17 +119,18 @@ def parse_json(data): return data if not sql: - raise ValueError(_(u"Bad SQL: %s" % sql)) + raise ValueError(_(f"Bad SQL: {sql}")) + params = [] if params is None else params cursor = connection.cursor() sql_params = fields + params if fields is not None else params if count: # do sql count of subquery, takes into account all options sql has and # is less hacky - sql = u"SELECT COUNT(*) FROM (" + sql + ") AS CQ" - fields = [u'count'] + sql = "SELECT COUNT(*) FROM (" + sql + ") AS CQ" + fields = ["count"] - cursor.execute(sql, [text(i) for i in sql_params]) + cursor.execute(sql, [str(i) for i in sql_params]) if fields is None: for row in cursor.fetchall(): yield parse_json(row[0]) if row[0] else None @@ -108,65 +140,67 @@ def parse_json(data): def get_etag_hash_from_query(queryset, sql=None, params=None): - """Returns md5 hash from the date_modified field or - """ + """Returns md5 hash from the date_modified field or""" if not isinstance(queryset, EmptyQuerySet): if sql is None: sql, params = queryset.query.sql_with_params() sql = ( "SELECT md5(string_agg(date_modified::text, ''))" - " FROM (SELECT date_modified " + sql[sql.find('FROM '):] + ") AS A" + " FROM (SELECT date_modified " + sql[sql.find("FROM ") :] + ") AS A" ) - etag_hash = [i for i in _query_iterator(sql, params=params) - if i is not None] + etag_hash = [i for i in _query_iterator(sql, params=params) if i is not None] if etag_hash: return etag_hash[0] - return u'%s' % datetime.datetime.utcnow() + return f"{datetime.datetime.utcnow()}" +# pylint: disable=too-many-arguments def _start_index_limit(records, sql, fields, params, sort, start_index, limit): - if start_index is not None and \ - (start_index < 0 or (limit is not None and limit < 0)): + if (start_index is not None and start_index < 0) or ( + limit is not None and limit < 0 + ): raise ValueError(_("Invalid start/limit params")) if (start_index is not None or limit is not None) and not sql: sql, params = records.query.sql_with_params() params = list(params) - start_index = 0 \ - if limit is not None and start_index is None else start_index - - if start_index is not None and \ - (ParsedInstance._has_json_fields(sort) or fields): + start_index = 0 if limit is not None and start_index is None else start_index + # pylint: disable=protected-access + has_json_fields = ParsedInstance._has_json_fields(sort) + if start_index is not None and (has_json_fields or fields): params += [start_index] - sql = u"%s OFFSET %%s" % sql - if limit is not None and \ - (ParsedInstance._has_json_fields(sort) or fields): - sql = u"%s LIMIT %%s" % sql + sql = f"{sql} OFFSET %%s" + if limit is not None and (has_json_fields or fields): + sql = f"{sql} LIMIT %%s" params += [limit] - if start_index is not None and limit is not None and not fields and \ - not ParsedInstance._has_json_fields(sort): - records = records[start_index: start_index + limit] - if start_index is not None and limit is None and not fields and \ - not ParsedInstance._has_json_fields(sort): + if ( + start_index is not None + and limit is not None + and not fields + and not has_json_fields + ): + records = records[start_index : start_index + limit] + if start_index is not None and limit is None and not fields and not has_json_fields: records = records[start_index:] return records, sql, params def _get_instances(xform, start, end): - kwargs = {'deleted_at': None} + kwargs = {"deleted_at": None} if isinstance(start, datetime.datetime): - kwargs.update({'date_created__gte': start}) + kwargs.update({"date_created__gte": start}) if isinstance(end, datetime.datetime): - kwargs.update({'date_created__lte': end}) + kwargs.update({"date_created__lte": end}) if xform.is_merged_dataset: - xforms = xform.mergedxform.xforms.filter(deleted_at__isnull=True)\ - .values_list('id', flat=True) - xform_ids = [i for i in xforms] or [xform.pk] + xforms = xform.mergedxform.xforms.filter(deleted_at__isnull=True).values_list( + "id", flat=True + ) + xform_ids = list(xforms) or [xform.pk] instances = Instance.objects.filter(xform_id__in=xform_ids) else: instances = xform.instances @@ -175,14 +209,27 @@ def _get_instances(xform, start, end): def _get_sort_fields(sort): - sort = ['id'] if sort is None else sort_from_mongo_sort_str(sort) - - return [i for i in _parse_sort_fields(sort)] - - -def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, - end=None, start_index=None, limit=None, count=None, - json_only: bool = True): + sort = ["id"] if sort is None else sort_from_mongo_sort_str(sort) + + return list(_parse_sort_fields(sort)) + + +# pylint: disable=too-many-locals +def get_sql_with_params( + xform, + query=None, + fields=None, + sort=None, + start=None, + end=None, + start_index=None, + limit=None, + count=None, + json_only: bool = True, +): + """ + Returns the SQL and related parameters. + """ records = _get_instances(xform, start, end) params = [] sort = _get_sort_fields(sort) @@ -190,31 +237,31 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, known_integers = [ get_name_from_survey_element(e) - for e in xform.get_survey_elements_of_type('integer')] + for e in xform.get_survey_elements_of_type("integer") + ] where, where_params = get_where_clause(query, known_integers) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) if fields: - field_list = [u"json->%s" for _i in fields] - sql = u"SELECT %s FROM logger_instance" % u",".join(field_list) + field_list = ["json->%s" for _i in fields] + sql = f"SELECT {','.join(field_list)} FROM logger_instance" - sql_where = u"" + sql_where = "" if where_params: - sql_where = u" AND " + u" AND ".join(where) + sql_where = " AND " + " AND ".join(where) - sql += u" WHERE xform_id = %s " + sql_where \ - + u" AND deleted_at IS NULL" + sql += " WHERE xform_id = %s " + sql_where + " AND deleted_at IS NULL" params = [xform.pk] + where_params else: if json_only: - records = records.values_list('json', flat=True) + records = records.values_list("json", flat=True) if query and isinstance(query, list): for qry in query: - w, wp = get_where_clause(qry, known_integers) - records = records.extra(where=w, params=wp) + _where, _where_params = get_where_clause(qry, known_integers) + records = records.extra(where=_where, params=_where_params) else: if where_params: @@ -222,18 +269,21 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, # apply sorting if not count and sort: + # pylint: disable=protected-access if ParsedInstance._has_json_fields(sort): if not fields: # we have to do a sql query for json field order sql, params = records.query.sql_with_params() params = list(params) + json_order_by_params( - sort, none_json_fields=NONE_JSON_FIELDS) - sql = u"%s %s" % (sql, json_order_by( - sort, - none_json_fields=NONE_JSON_FIELDS, - model_name="logger_instance")) - elif not fields: - records = records.order_by(*sort) + sort, none_json_fields=NONE_JSON_FIELDS + ) + _json_order_by = json_order_by( + sort, none_json_fields=NONE_JSON_FIELDS, model_name="logger_instance" + ) + sql = f"{sql} {_json_order_by}" + else: + if not fields: + records = records.order_by(*sort) records, sql, params = _start_index_limit( records, sql, fields, params, sort, start_index, limit @@ -242,36 +292,60 @@ def get_sql_with_params(xform, query=None, fields=None, sort=None, start=None, return sql, params, records -def query_data(xform, query=None, fields=None, sort=None, start=None, - end=None, start_index=None, limit=None, count=None, - json_only: bool = True): +def query_data( + xform, + query=None, + fields=None, + sort=None, + start=None, + end=None, + start_index=None, + limit=None, + count=None, + json_only: bool = True, +): sql, params, records = get_sql_with_params( - xform, query, fields, sort, start, end, start_index, limit, count, - json_only=json_only + xform, + query, + fields, + sort, + start, + end, + start_index, + limit, + count, + json_only=json_only, ) if fields and isinstance(fields, six.string_types): fields = json.loads(fields) sort = _get_sort_fields(sort) + # pylint: disable=protected-access if (ParsedInstance._has_json_fields(sort) or fields) and sql: records = _query_iterator(sql, fields, params, count) if count and isinstance(records, types.GeneratorType): - return [i for i in records] - elif count: + return list(records) + if count: return [{"count": records.count()}] return records class ParsedInstance(models.Model): - USERFORM_ID = u'_userform_id' - STATUS = u'_status' + """ + ParsedInstance - parsed XML submission, represents the XML submissions as a python + object. + """ + + USERFORM_ID = "_userform_id" + STATUS = "_status" DEFAULT_LIMIT = settings.PARSED_INSTANCE_DEFAULT_LIMIT DEFAULT_BATCHSIZE = settings.PARSED_INSTANCE_DEFAULT_BATCHSIZE instance = models.OneToOneField( - Instance, related_name="parsed_instance", on_delete=models.CASCADE) + Instance, related_name="parsed_instance", on_delete=models.CASCADE + ) start_time = models.DateTimeField(null=True) end_time = models.DateTimeField(null=True) # TODO: decide if decimal field is better than float field. @@ -288,64 +362,70 @@ def _has_json_fields(cls, sort_list): """ fields = [f.name for f in Instance._meta.get_fields()] - return any([i for i in sort_list if i.lstrip('-') not in fields]) + return any(i for i in sort_list if i.lstrip("-") not in fields) def to_dict_for_mongo(self): - d = self.to_dict() + """ + Return the XForm XML submission as a python object. + """ + data_dict = self.to_dict() data = { UUID: self.instance.uuid, ID: self.instance.id, BAMBOO_DATASET_ID: self.instance.xform.bamboo_dataset, - self.USERFORM_ID: u'%s_%s' % ( - self.instance.xform.user.username, - self.instance.xform.id_string), + self.USERFORM_ID: ( + f"{self.instance.xform.user.username}_" + f"{self.instance.xform.id_string}" + ), ATTACHMENTS: _get_attachments_from_instance(self.instance), self.STATUS: self.instance.status, GEOLOCATION: [self.lat, self.lng], - SUBMISSION_TIME: self.instance.date_created.strftime( - MONGO_STRFTIME), - DATE_MODIFIED: self.instance.date_modified.strftime( - MONGO_STRFTIME), + SUBMISSION_TIME: self.instance.date_created.strftime(MONGO_STRFTIME), + DATE_MODIFIED: self.instance.date_modified.strftime(MONGO_STRFTIME), TAGS: list(self.instance.tags.names()), NOTES: self.get_notes(), - SUBMITTED_BY: self.instance.user.username - if self.instance.user else None, + SUBMITTED_BY: self.instance.user.username if self.instance.user else None, VERSION: self.instance.version, DURATION: self.instance.get_duration(), XFORM_ID: self.instance.xform.pk, TOTAL_MEDIA: self.instance.total_media, MEDIA_COUNT: self.instance.media_count, - MEDIA_ALL_RECEIVED: self.instance.media_all_received + MEDIA_ALL_RECEIVED: self.instance.media_all_received, } if isinstance(self.instance.deleted_at, datetime.datetime): data[DELETEDAT] = self.instance.deleted_at.strftime(MONGO_STRFTIME) if self.instance.has_a_review: - review = self.get_latest_review() + review = self.instance.get_latest_review() if review: data[REVIEW_STATUS] = review.status - data[REVIEW_DATE] = review.date_created.strftime( - MONGO_STRFTIME) + data[REVIEW_DATE] = review.date_created.strftime(MONGO_STRFTIME) if review.get_note_text(): data[REVIEW_COMMENT] = review.get_note_text() - data[EDITED] = (True if self.instance.submission_history.count() > 0 - else False) + data[EDITED] = self.instance.submission_history.count() > 0 - d.update(data) + data_dict.update(data) - return dict_for_mongo(d) + return dict_for_mongo(data_dict) def to_dict(self): + """ + Returns a python dictionary object of a submission. + """ if not hasattr(self, "_dict_cache"): + # pylint: disable=attribute-defined-outside-init self._dict_cache = self.instance.get_dict() return self._dict_cache @classmethod def dicts(cls, xform): - qs = cls.objects.filter(instance__xform=xform) - for parsed_instance in queryset_iterator(qs): + """ + Iterates over a forms submissions. + """ + queryset = cls.objects.filter(instance__xform=xform) + for parsed_instance in queryset_iterator(queryset): yield parsed_instance.to_dict() def _get_name_for_type(self, type_value): @@ -358,9 +438,10 @@ def _get_name_for_type(self, type_value): type_value ('start' or 'end') """ datadict = json.loads(self.instance.xform.json) - for item in datadict['children']: - if isinstance(item, dict) and item.get(u'type') == type_value: - return item['name'] + for item in datadict["children"]: + if isinstance(item, dict) and item.get("type") == type_value: + return item["name"] + return None # TODO: figure out how much of this code should be here versus # data_dictionary.py. @@ -375,24 +456,29 @@ def save(self, *args, **kwargs): # noqa self.start_time = None self.end_time = None self._set_geopoint() - super(ParsedInstance, self).save(*args, **kwargs) # noqa + super().save(*args, **kwargs) # noqa def add_note(self, note): + """ + Add a note for the instance. + """ note = Note(instance=self.instance, note=note) note.save() - def remove_note(self, pk): - note = self.instance.notes.get(pk=pk) + def remove_note(self, note_id): + """ + Deletes the note with the `pk` as ``note_id`` + """ + note = self.instance.notes.get(pk=note_id) note.delete() def get_notes(self): notes = [] note_qs = self.instance.notes.values( - 'id', 'note', 'date_created', 'date_modified') + "id", "note", "date_created", "date_modified" + ) for note in note_qs: - note['date_created'] = note['date_created'].strftime( - MONGO_STRFTIME) - note['date_modified'] = note['date_modified'].strftime( - MONGO_STRFTIME) + note["date_created"] = note["date_created"].strftime(MONGO_STRFTIME) + note["date_modified"] = note["date_modified"].strftime(MONGO_STRFTIME) notes.append(note) return notes diff --git a/onadata/apps/viewer/parsed_instance_tools.py b/onadata/apps/viewer/parsed_instance_tools.py index d166b6ed11..eca1828f5a 100644 --- a/onadata/apps/viewer/parsed_instance_tools.py +++ b/onadata/apps/viewer/parsed_instance_tools.py @@ -1,10 +1,15 @@ -import json -import six +# -*- coding: utf-8 -*- +""" +ParsedInstance model utility functions +""" import datetime +import json from builtins import str as text from typing import Any, Tuple -from onadata.libs.utils.common_tags import MONGO_STRFTIME, DATE_FORMAT +import six + +from onadata.libs.utils.common_tags import DATE_FORMAT, MONGO_STRFTIME KNOWN_DATES = ["_submission_time"] NONE_JSON_FIELDS = { @@ -14,6 +19,7 @@ "_version": "version", "_last_edited": "last_edited", } +OPERANDS = {"$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<=", "$i": "~*"} def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=None): @@ -35,11 +41,12 @@ def _json_sql_str(key, known_integers=None, known_dates=None, known_decimals=Non return _json_str +# pylint: disable=too-many-locals,too-many-branches def _parse_where(query, known_integers, known_decimals, or_where, or_params): # using a dictionary here just incase we will need to filter using # other table columns where, where_params = [], [] - OPERANDS = {"$gt": ">", "$gte": ">=", "$lt": "<", "$lte": "<=", "$i": "~*"} + # pylint: disable=too-many-nested-blocks for (field_key, field_value) in six.iteritems(query): if isinstance(field_value, dict): if field_key in NONE_JSON_FIELDS: @@ -66,7 +73,7 @@ def _parse_where(query, known_integers, known_decimals, or_where, or_params): where_params.extend((field_key, text(_v))) else: if field_key in NONE_JSON_FIELDS: - where.append("{} = %s".format(NONE_JSON_FIELDS[field_key])) + where.append(f"{NONE_JSON_FIELDS[field_key]} = %s") where_params.extend([text(field_value)]) elif field_value is None: where.append(f"json->>'{field_key}' IS NULL") @@ -93,6 +100,9 @@ def _merge_duplicate_keys(pairs: Tuple[str, Any]): def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): + """ + Returns where clause and related parameters. + """ if form_integer_fields is None: form_integer_fields = [] if form_decimal_fields is None: @@ -102,6 +112,7 @@ def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): where = [] where_params = [] + # pylint: disable=too-many-nested-blocks try: if query and isinstance(query, (dict, six.string_types)): query = ( @@ -120,7 +131,7 @@ def get_where_clause(query, form_integer_fields=None, form_decimal_fields=None): for or_query in or_dict: for k, v in or_query.items(): if v is None: - or_where.extend(["json->>'{}' IS NULL".format(k)]) + or_where.extend([f"json->>'{k}' IS NULL"]) elif isinstance(v, list): for value in v: or_where.extend(["json->>%s = %s"]) diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 30e1e3d597..9dcb3761ff 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -183,7 +183,7 @@ def _get_metadata(self, obj, key): if m.data_type == key: return m.data_value - return obj.metadata_set.all() + return None # pylint: disable=no-self-use def get_users(self, obj): From 010c503caa8a8588c4ef6623fe137736d1c77d41 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 04:57:29 +0300 Subject: [PATCH 083/234] batch: cleanup --- onadata/apps/main/models/meta_data.py | 140 ++++++++++++++++--- onadata/libs/renderers/renderers.py | 51 ++++--- onadata/libs/serializers/chart_serializer.py | 31 ++-- 3 files changed, 172 insertions(+), 50 deletions(-) diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index aab4dbf687..8b1261b14d 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +MetaData model +""" from __future__ import unicode_literals import logging @@ -6,9 +10,7 @@ from contextlib import closing from hashlib import md5 -import requests from django.conf import settings -from django.utils import timezone from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError @@ -17,13 +19,16 @@ from django.core.validators import URLValidator from django.db import IntegrityError, models from django.db.models.signals import post_delete, post_save +from django.utils import timezone + +import requests from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE, safe_delete from onadata.libs.utils.common_tags import ( GOOGLE_SHEET_DATA_TYPE, TEXTIT, - XFORM_META_PERMS, TEXTIT_DETAILS, + XFORM_META_PERMS, ) CHUNK_SIZE = 1024 @@ -38,6 +43,9 @@ def is_valid_url(uri): + """ + Validates a URI. + """ try: urlvalidate(uri) except ValidationError: @@ -47,6 +55,9 @@ def is_valid_url(uri): def upload_to(instance, filename): + """ + Returns the upload path for given ``filename``. + """ username = None if ( @@ -64,16 +75,22 @@ def upload_to(instance, filename): def save_metadata(metadata_obj): + """ + Saves the MetaData object and returns it. + """ try: metadata_obj.save() except IntegrityError: - logging.exception("MetaData object '%s' already exists" % metadata_obj) + logging.exception("MetaData object '%s' already exists", metadata_obj) return metadata_obj def get_default_content_type(): - content_object, created = ContentType.objects.get_or_create( + """ + Returns the default content type id for the XForm model. + """ + content_object, _created = ContentType.objects.get_or_create( app_label="logger", model=XFORM_MODEL_NAME ) @@ -94,7 +111,7 @@ def unique_type_for_form(content_object, data_type, data_value=None, data_file=N object_id=content_object.id, content_type=content_type, data_type=data_type ).first() else: - result, created = MetaData.objects.update_or_create( + result, _created = MetaData.objects.update_or_create( object_id=content_object.id, content_type=content_type, data_type=data_type, @@ -112,6 +129,9 @@ def unique_type_for_form(content_object, data_type, data_value=None, data_file=N def type_for_form(content_object, data_type): + """ + Returns the MetaData queryset for ``content_object`` of the given ``data_type``. + """ content_type = ContentType.objects.get_for_model(content_object) return MetaData.objects.filter( object_id=content_object.id, content_type=content_type, data_type=data_type @@ -124,8 +144,8 @@ def create_media(media): filename = media.data_value.split("/")[-1] data_file = NamedTemporaryFile() content_type = mimetypes.guess_type(filename) - with closing(requests.get(media.data_value, stream=True)) as r: - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + with closing(requests.get(media.data_value, stream=True)) as resp: + for chunk in resp.iter_content(chunk_size=CHUNK_SIZE): if chunk: data_file.write(chunk) data_file.seek(os.SEEK_SET, os.SEEK_END) @@ -164,6 +184,7 @@ def media_resources(media_list, download=False): return data +# pylint: disable=too-many-public-methods class MetaData(models.Model): data_type = models.CharField(max_length=255) data_value = models.CharField(max_length=255) @@ -187,16 +208,22 @@ class Meta: def save(self, *args, **kwargs): self._set_hash() - super(MetaData, self).save(*args, **kwargs) + super().save(*args, **kwargs) @property def hash(self): + """ + Returns the md5 hash of the metadata file. + """ if self.file_hash is not None and self.file_hash != "": return self.file_hash - else: - return self._set_hash() + + return self._set_hash() def _set_hash(self): + """ + Returns the md5 hash of the metadata file. + """ if not self.data_file: return None @@ -210,7 +237,7 @@ def _set_hash(self): except IOError: return "" else: - self.file_hash = "md5:%s" % md5(self.data_file.read()).hexdigest() + self.file_hash = f"md5:{md5(self.data_file.read()).hexdigest()}" return self.file_hash @@ -265,52 +292,78 @@ def get_google_sheet_details(obj): # {'A': 'a', 'B': 'b c'} return dict([tuple(a.strip().split(" ", 1)) for a in data_list]) + return metadata_data_value + @staticmethod def published_by_formbuilder(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'published_by_formbuilder' + """ data_type = "published_by_formbuilder" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_url(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'enket_url' + """ data_type = "enketo_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_preview_url(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'enketo_preview_url' + """ data_type = "enketo_preview_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def enketo_single_submit_url(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'enketo_single_submit_url' + """ data_type = "enketo_single_submit_url" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def form_license(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'form_license' + """ data_type = "form_license" obj = unique_type_for_form(content_object, data_type, data_value) - return (obj and obj.data_value) or None + return obj.data_value if obj else None @staticmethod def data_license(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'data_license' + """ data_type = "data_license" obj = unique_type_for_form(content_object, data_type, data_value) - return (obj and obj.data_value) or None + return obj.data_value if obj else None @staticmethod def source(content_object, data_value=None, data_file=None): + """ + Returns the metadata object where data_type is 'source' + """ data_type = "source" return unique_type_for_form(content_object, data_type, data_value, data_file) @staticmethod def supporting_docs(content_object, data_file=None): + """ + Returns the metadata object where data_type is 'supporting_doc' + """ data_type = "supporting_doc" if data_file: content_type = ContentType.objects.get_for_model(content_object) - doc, created = MetaData.objects.update_or_create( + _doc, _created = MetaData.objects.update_or_create( data_type=data_type, content_type=content_type, object_id=content_object.id, @@ -325,6 +378,9 @@ def supporting_docs(content_object, data_file=None): @staticmethod def media_upload(content_object, data_file=None, download=False): + """ + Returns the metadata object where data_type is 'media' + """ data_type = "media" if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES @@ -337,7 +393,7 @@ def media_upload(content_object, data_file=None, download=False): if data_content_type in allowed_types: content_type = ContentType.objects.get_for_model(content_object) - media, created = MetaData.objects.update_or_create( + _media, _created = MetaData.objects.update_or_create( data_type=data_type, content_type=content_type, object_id=content_object.id, @@ -355,7 +411,7 @@ def media_add_uri(content_object, uri): data_type = "media" if is_valid_url(uri): - media, created = MetaData.objects.update_or_create( + _media, _created = MetaData.objects.update_or_create( data_type=data_type, data_value=uri, defaults={ @@ -365,20 +421,23 @@ def media_add_uri(content_object, uri): @staticmethod def mapbox_layer_upload(content_object, data=None): + """ + Returns the metadata object where data_type is 'mapbox_layer' + """ data_type = "mapbox_layer" if data and not MetaData.objects.filter( object_id=content_object.id, data_type="mapbox_layer" ): - s = "" + data_value = "" for key in data: - s = s + data[key] + "||" + data_value = data_value + data[key] + "||" content_type = ContentType.objects.get_for_model(content_object) mapbox_layer = MetaData( data_type=data_type, content_type=content_type, object_id=content_object.id, - data_value=s, + data_value=data_value, ) mapbox_layer.save() if type_for_form(content_object, data_type): @@ -389,11 +448,14 @@ def mapbox_layer_upload(content_object, data=None): data_values["attribution"] = values[2] data_values["id"] = type_for_form(content_object, data_type)[0].id return data_values - else: - return None + + return None @staticmethod def external_export(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'external_export' + """ data_type = "external_export" if data_value: @@ -413,18 +475,27 @@ def external_export(content_object, data_value=None): @property def external_export_url(self): + """ + Returns the external export URL + """ parts = self.data_value.split("|") return parts[1] if len(parts) > 1 else None @property def external_export_name(self): + """ + Returns the external export name + """ parts = self.data_value.split("|") return parts[0] if len(parts) > 1 else None @property def external_export_template(self): + """ + Returns the exxernal export, "XLS report", template + """ parts = self.data_value.split("|") return parts[1].replace("xls", "templates") if len(parts) > 1 else None @@ -439,11 +510,17 @@ def textit(content_object, data_value=None): @staticmethod def textit_flow_details(content_object, data_value: str = ""): + """ + Returns the metadata object where data_type is 'textit_details' + """ data_type = TEXTIT_DETAILS return unique_type_for_form(content_object, data_type, data_value) @property def is_linked_dataset(self): + """ + Returns True if the metadata object is a linked dataset. + """ return isinstance(self.data_value, str) and ( self.data_value.startswith("xform") or self.data_value.startswith("dataview") @@ -451,28 +528,45 @@ def is_linked_dataset(self): @staticmethod def xform_meta_permission(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'xform_meta_perms' + """ data_type = XFORM_META_PERMS return unique_type_for_form(content_object, data_type, data_value) @staticmethod def submission_review(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'submission_review' + """ data_type = "submission_review" return unique_type_for_form(content_object, data_type, data_value) @staticmethod def instance_csv_imported_by(content_object, data_value=None): + """ + Returns the metadata object where data_type is 'imported_via_csv_by' + """ data_type = "imported_via_csv_by" return unique_type_for_form(content_object, data_type, data_value) +# pylint: disable=unused-argument,invalid-name def clear_cached_metadata_instance_object( sender, instance=None, created=False, **kwargs ): - safe_delete("{}{}".format(XFORM_METADATA_CACHE, instance.object_id)) + """ + Clear the cache for the metadata object. + """ + safe_delete(f"{XFORM_METADATA_CACHE}{instance.object_id}") +# pylint: disable=unused-argument def update_attached_object(sender, instance=None, created=False, **kwargs): + """ + Save the content_object attached to a MetaData instance. + """ if instance: instance.content_object.save() diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index eb099775f4..3c4833736d 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -80,19 +80,39 @@ def floip_list(data): yield i +def _pop_xml_attributes(xml_dictionary: dict) -> Tuple[dict, dict]: + """ + Extracts XML attributes from the ``xml_dictionary``. + """ + ret = xml_dictionary.copy() + attributes = {} + + for key, value in xml_dictionary.items(): + if key.startswith("@"): + attributes.update({key.replace("@", ""): value}) + del ret[key] + + return ret, attributes + + class DecimalEncoder(JSONEncoder): """ JSON DecimalEncoder that returns None for decimal nan json values. """ - def default(self, obj): # pylint: disable=method-hidden + # pylint: disable=method-hidden + def default(self, obj): + """ + JSON DecimalEncoder that returns None for decimal nan json values. + """ # Handle Decimal NaN values if isinstance(obj, decimal.Decimal) and math.isnan(obj): return None return JSONEncoder.default(self, obj) -class XLSRenderer(BaseRenderer): # pylint: disable=too-few-public-methods +# pylint: disable=abstract-method,too-few-public-methods +class XLSRenderer(BaseRenderer): """ XLSRenderer - renders .xls spreadsheet documents with application/vnd.openxmlformats. @@ -101,8 +121,11 @@ class XLSRenderer(BaseRenderer): # pylint: disable=too-few-public-methods media_type = "application/vnd.openxmlformats" format = "xls" charset = None - + # pylint: disable=no-self-use,unused-argument def render(self, data, accepted_media_type=None, renderer_context=None): + """ + Encode ``data`` string to 'utf-8'. + """ if isinstance(data, six.text_type): return data.encode("utf-8") return data @@ -202,7 +225,7 @@ class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): matching format. """ - # pylint: disable=redefined-builtin + # pylint: disable=redefined-builtin,no-self-use def filter_renderers(self, renderers, format): """ If there is a '.json' style format suffix, filter the renderers @@ -339,6 +362,7 @@ def stream_data(self, data, serializer): if data is None: yield "" + # pylint: disable=attribute-defined-outside-init self.stream = StringIO() xml = SimplerXMLGenerator(self.stream, self.charset) @@ -358,7 +382,7 @@ def stream_data(self, data, serializer): try: next_item = next(data) out = serializer(out).data - out, attributes = self._pop_xml_attributes(out) + out, attributes = _pop_xml_attributes(out) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, out) xml.endElement(self.item_tag_name) @@ -366,7 +390,7 @@ def stream_data(self, data, serializer): out = next_item except StopIteration: out = serializer(out).data - out, attributes = self._pop_xml_attributes(out) + out, attributes = _pop_xml_attributes(out) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, out) xml.endElement(self.item_tag_name) @@ -398,21 +422,10 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return stream.getvalue() - def _pop_xml_attributes(self, xml_dictionary: dict) -> Tuple[dict, dict]: - ret = xml_dictionary.copy() - attributes = {} - - for key, value in xml_dictionary.items(): - if key.startswith("@"): - attributes.update({key.replace("@", ""): value}) - del ret[key] - - return ret, attributes - def _to_xml(self, xml, data): if isinstance(data, (list, tuple)): for item in data: - item, attributes = self._pop_xml_attributes(item) + item, attributes = _pop_xml_attributes(item) xml.startElement(self.item_tag_name, attributes) self._to_xml(xml, item) xml.endElement(self.item_tag_name) @@ -428,7 +441,7 @@ def _to_xml(self, xml, data): xml.endElement(key) elif isinstance(value, dict): - value, attributes = self._pop_xml_attributes(value) + value, attributes = _pop_xml_attributes(value) xml.startElement(key, attributes) self._to_xml(xml, value) xml.endElement(key) diff --git a/onadata/libs/serializers/chart_serializer.py b/onadata/libs/serializers/chart_serializer.py index 69b1f7b41d..3e2f874fb0 100644 --- a/onadata/libs/serializers/chart_serializer.py +++ b/onadata/libs/serializers/chart_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Chart serializer. +""" from django.http import Http404 from rest_framework import serializers @@ -7,7 +11,12 @@ from onadata.libs.utils.common_tags import INSTANCE_ID +# pylint: disable=too-many-public-methods class ChartSerializer(serializers.HyperlinkedModelSerializer): + """ + Chart serializer + """ + url = serializers.HyperlinkedIdentityField( view_name="chart-detail", lookup_field="pk" ) @@ -17,33 +26,39 @@ class Meta: fields = ("id", "id_string", "url") +# pylint: disable=too-many-public-methods class FieldsChartSerializer(serializers.ModelSerializer): + """ + Generate chart data for the field. + """ + class Meta: model = XForm - def to_representation(self, obj): + def to_representation(self, instance): + """ + Generate chart data for a given field in the request query params. + """ data = {} request = self.context.get("request") - if obj is not None: - fields = obj.survey_elements + if instance is not None: + fields = instance.survey_elements if request: selected_fields = request.query_params.get("fields") if isinstance(selected_fields, str) and selected_fields != "all": fields = selected_fields.split(",") - fields = [e for e in obj.survey_elements if e.name in fields] + fields = [e for e in instance.survey_elements if e.name in fields] if len(fields) == 0: - raise Http404( - "Field %s does not not exist on the form" % fields - ) + raise Http404(f"Field {fields} does not not exist on the form") for field in fields: if field.name == INSTANCE_ID: continue - field_data = build_chart_data_for_field(obj, field) + field_data = build_chart_data_for_field(instance, field) data[field.name] = field_data return data From 2ec3ea1f97127185ec91c4244635d87a791af517 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 08:22:57 +0300 Subject: [PATCH 084/234] batch: cleanup --- .../commands/fix_readonly_role_perms.py | 121 ++++--- .../commands/increase_odk_token_lifetime.py | 45 ++- .../test_increase_odk_token_lifetime.py | 21 +- onadata/apps/api/viewsets/data_viewset.py | 328 ++++++++++-------- onadata/apps/api/viewsets/osm_viewset.py | 188 +++++----- onadata/apps/logger/models/osmdata.py | 14 +- onadata/apps/viewer/models/parsed_instance.py | 4 +- 7 files changed, 409 insertions(+), 312 deletions(-) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 265f459ca6..1114bc4754 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -54,10 +54,66 @@ def check_role(role_class, user, obj, new_perm=None): return False +def reassign_perms(user, model, new_perm): + """ + Gets all the permissions the user has on objects and assigns the new + permission to them + :param user: + :param model: + :param new_perm: + :return: + """ + + # Get the unique permission model objects filtered by content type + # for the user + if isinstance(user, Team): + if model == "project": + objects = user.projectgroupobjectpermission_set.filter( + group_id=user.pk + ).distinct("content_object_id") + else: + objects = user.xformgroupobjectpermission_set.filter( + group_id=user.pk + ).distinct("content_object_id") + else: + if model == "project": + objects = user.projectuserobjectpermission_set.all() + else: + objects = user.xformuserobjectpermission_set.all() + + for perm_obj in objects: + obj = perm_obj.content_object + ROLES = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ] + + # For each role reassign the perms + for role_class in reversed(ROLES): + not_readonly = role_class.user_has_role(user, obj) or role_class not in [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + ] + if not_readonly: + continue + + if check_role(role_class, user, obj, new_perm): + # If true + role_class.add(user, obj) + break + + class Command(BaseCommand): """ fix_readonly_role_perms - Reassign permission to the model when - permissions are changed + permissions are changed """ args = "" @@ -88,68 +144,9 @@ def handle(self, *args, **options): teams = Team.objects.filter(organization__username=username) # Get all the users for user in queryset_iterator(users): - self.reassign_perms(user, app, model, new_perms) + reassign_perms(user, model, new_perms) for team in queryset_iterator(teams): - self.reassign_perms(team, app, model, new_perms) + reassign_perms(team, model, new_perms) self.stdout.write("Re-assigining finished", ending="\n") - - # pylint: disable=unused-argument - def reassign_perms(self, user, app, model, new_perm): - """ - Gets all the permissions the user has on objects and assigns the new - permission to them - :param user: - :param app: - :param model: - :param new_perm: - :return: - """ - - # Get the unique permission model objects filtered by content type - # for the user - if isinstance(user, Team): - if model == "project": - objects = user.projectgroupobjectpermission_set.filter( - group_id=user.pk - ).distinct("content_object_id") - else: - objects = user.xformgroupobjectpermission_set.filter( - group_id=user.pk - ).distinct("content_object_id") - else: - if model == "project": - objects = user.projectuserobjectpermission_set.all() - else: - objects = user.xformuserobjectpermission_set.all() - - for perm_obj in objects: - obj = perm_obj.content_object - ROLES = [ - ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryOnlyRole, - DataEntryMinorRole, - DataEntryRole, - EditorMinorRole, - EditorRole, - ManagerRole, - OwnerRole, - ] - - # For each role reassign the perms - for role_class in reversed(ROLES): - not_readonly = role_class.user_has_role( - user, obj - ) or role_class not in [ - ReadOnlyRoleNoDownload, - ReadOnlyRole, - ] - if not_readonly: - continue - - if check_role(role_class, user, obj, new_perm): - # If true - role_class.add(user, obj) - break diff --git a/onadata/apps/api/management/commands/increase_odk_token_lifetime.py b/onadata/apps/api/management/commands/increase_odk_token_lifetime.py index e98caeb687..d1e714719a 100644 --- a/onadata/apps/api/management/commands/increase_odk_token_lifetime.py +++ b/onadata/apps/api/management/commands/increase_odk_token_lifetime.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Command to increase the ODK token lifetime for a user. +""" from datetime import timedelta from django.core.management.base import BaseCommand, CommandError @@ -7,12 +11,14 @@ def increase_odk_token_lifetime(days: int, username: str): - qs = ODKToken.objects.filter( - user__username=username, status=ODKToken.ACTIVE) - if qs.count() < 1: + """ + Increase ODK Token lifetime for a particular user. + """ + queryset = ODKToken.objects.filter(user__username=username, status=ODKToken.ACTIVE) + if queryset.count() < 1: return False - token = qs.first() + token = queryset.first() updated_expiry_date = token.expires + timedelta(days=days) token.expires = updated_expiry_date.astimezone(token.expires.tzinfo) token.save() @@ -20,31 +26,34 @@ def increase_odk_token_lifetime(days: int, username: str): class Command(BaseCommand): + """ + increase_odk_token_lifetime command + + Increase ODK Token lifetime for a particular user. + """ + help = _("Increase ODK Token lifetime for a particular user.") def add_arguments(self, parser): parser.add_argument( - '--days', - '-d', + "--days", + "-d", default=30, - dest='days', - help='Number of days to increase the token lifetime by.') + dest="days", + help="Number of days to increase the token lifetime by.", + ) parser.add_argument( - '--username', - '-u', - dest='username', - help='The users username' + "--username", "-u", dest="username", help="The users username" ) def handle(self, *args, **options): - username = options.get('username') - days = options.get('days') + username = options.get("username") + days = options.get("days") if not username: - raise CommandError('No username provided.') + raise CommandError("No username provided.") created = increase_odk_token_lifetime(days, username) if not created: - raise CommandError(f'User {username} has no active ODK Token.') - self.stdout.write( - f'Increased the lifetime of ODK Token for user {username}') + raise CommandError(f"User {username} has no active ODK Token.") + self.stdout.write(f"Increased the lifetime of ODK Token for user {username}") diff --git a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py index af54161d35..c4936b72d0 100644 --- a/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py +++ b/onadata/apps/api/tests/management/commands/test_increase_odk_token_lifetime.py @@ -1,16 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Test increase_odk_token_lifetime command +""" from datetime import timedelta -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management import call_command + from six import StringIO -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.models.odk_token import ODKToken +from onadata.apps.main.tests.test_base import TestBase class IncreaseODKTokenLifetimeTest(TestBase): + """ + Test increase_odk_token_lifetime command + """ + + # pylint: disable=invalid-name def test_increase_odk_token_lifetime(self): - user = User.objects.create(username="dave", email="dave@example.com") + """ + Test increase_odk_token_lifetime command + """ + user = get_user_model().objects.create( + username="dave", email="dave@example.com" + ) token = ODKToken.objects.create(user=user) expiry_date = token.expires diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index f6dcba0e54..6c05e76a5d 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -1,20 +1,23 @@ +# -*- coding: utf-8 -*- +""" +The /data API endpoint. +""" import json +import math import types from builtins import str as text from typing import Union -import six - from django.conf import settings from django.core.exceptions import PermissionDenied from django.db.models import Q from django.db.models.query import QuerySet from django.db.utils import DataError, OperationalError -from django.http import Http404 -from django.http import StreamingHttpResponse +from django.http import Http404, StreamingHttpResponse from django.utils import timezone -from distutils.util import strtobool from django.utils.translation import ugettext as _ + +import six from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import ParseError @@ -23,24 +26,23 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet -from onadata.apps.api.permissions import ConnectViewsetPermissions -from onadata.apps.api.permissions import XFormPermissions -from onadata.apps.api.tools import add_tags_to_instance -from onadata.apps.api.tools import get_baseviewset_class -from onadata.apps.logger.models import OsmData, MergedXForm +from onadata.apps.api.permissions import ConnectViewsetPermissions, XFormPermissions +from onadata.apps.api.tools import add_tags_to_instance, get_baseviewset_class +from onadata.apps.logger.models import MergedXForm, OsmData from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.instance import Instance, FormInactiveError +from onadata.apps.logger.models.instance import FormInactiveError, Instance from onadata.apps.logger.models.xform import XForm -from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED +from onadata.apps.messaging.constants import SUBMISSION_DELETED, XFORM from onadata.apps.messaging.serializers import send_message -from onadata.apps.viewer.models.parsed_instance import get_etag_hash_from_query -from onadata.apps.viewer.models.parsed_instance import get_sql_with_params -from onadata.apps.viewer.models.parsed_instance import get_where_clause -from onadata.apps.viewer.models.parsed_instance import query_data +from onadata.apps.viewer.models.parsed_instance import ( + get_etag_hash_from_query, + get_sql_with_params, + get_where_clause, + query_data, +) from onadata.libs import filters from onadata.libs.data import parse_int -from onadata.libs.exceptions import EnketoError -from onadata.libs.exceptions import NoRecordsPermission +from onadata.libs.exceptions import EnketoError, NoRecordsPermission from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( AnonymousUserPublicFormsMixin, ) @@ -57,15 +59,15 @@ from onadata.libs.serializers.data_serializer import ( DataInstanceSerializer, DataInstanceXMLSerializer, + DataSerializer, InstanceHistorySerializer, + JsonDataSerializer, + OSMSerializer, ) -from onadata.libs.serializers.data_serializer import DataSerializer -from onadata.libs.serializers.data_serializer import JsonDataSerializer -from onadata.libs.serializers.data_serializer import OSMSerializer from onadata.libs.serializers.geojson_serializer import GeoJsonSerializer from onadata.libs.utils.api_export_tools import custom_response_handler from onadata.libs.utils.common_tools import json_stream -from onadata.libs.utils.viewer_tools import get_form_url, get_enketo_urls +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] SUBMISSION_RETRIEVAL_THRESHOLD = getattr( @@ -75,7 +77,26 @@ BaseViewset = get_baseviewset_class() +# source from deprecated module distutils/util.py +def strtobool(val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return 1 + if val in ("n", "no", "f", "false", "off", "0"): + return 0 + raise ValueError(f"invalid truth value {val}") + + def get_data_and_form(kwargs): + """ + Checks if the dataid in ``kwargs`` is a valid integer. + """ data_id = text(kwargs.get("dataid")) if not data_id.isdigit(): raise ParseError(_("Data ID should be an integer")) @@ -94,9 +115,10 @@ def delete_instance(instance, user): try: instance.set_deleted(timezone.now(), user) except FormInactiveError as e: - raise ParseError(text(e)) + raise ParseError(text(e)) from e +# pylint: disable=too-many-ancestors class DataViewSet( AnonymousUserPublicFormsMixin, AuthenticateHeaderMixin, @@ -141,7 +163,7 @@ class DataViewSet( def get_serializer_class(self): pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) fmt = self.kwargs.get("format", self.request.GET.get("format")) sort = self.request.GET.get("sort") @@ -152,35 +174,40 @@ def get_serializer_class(self): serializer_class = GeoJsonSerializer elif fmt == "xml": serializer_class = DataInstanceXMLSerializer - elif pk is not None and dataid is None and pk != self.public_data_endpoint: + elif ( + form_pk is not None + and dataid is None + and form_pk != self.public_data_endpoint + ): if sort or fields: serializer_class = JsonDataSerializer else: serializer_class = DataInstanceSerializer else: - serializer_class = super(DataViewSet, self).get_serializer_class() + serializer_class = super().get_serializer_class() return serializer_class + # pylint: disable=unused-argument def get_object(self, queryset=None): - obj = super(DataViewSet, self).get_object() + obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - if pk is not None and dataid is not None: + if form_pk is not None and dataid is not None: try: int(dataid) - except ValueError: - raise ParseError(_("Invalid dataid %(dataid)s" % {"dataid": dataid})) + except ValueError as e: + raise ParseError(_(f"Invalid dataid {dataid}")) from e if not obj.is_merged_dataset: obj = get_object_or_404( - Instance, pk=dataid, xform__pk=pk, deleted_at__isnull=True + Instance, pk=dataid, xform__pk=form_pk, deleted_at__isnull=True ) else: xforms = obj.mergedxform.xforms.filter(deleted_at__isnull=True) - pks = [xform_id for xform_id in xforms.values_list("pk", flat=True)] + pks = list(xforms.values_list("pk", flat=True)) obj = get_object_or_404( Instance, pk=dataid, xform_id__in=pks, deleted_at__isnull=True @@ -193,45 +220,46 @@ def _get_public_forms_queryset(self): Q(shared=True) | Q(shared_data=True), deleted_at__isnull=True ) - def _filtered_or_shared_qs(self, qs, pk): - filter_kwargs = {self.lookup_field: pk} - qs = qs.filter(**filter_kwargs).only("id", "shared") + def _filtered_or_shared_queryset(self, queryset, form_pk): + filter_kwargs = {self.lookup_field: form_pk} + queryset = queryset.filter(**filter_kwargs).only("id", "shared") - if not qs: + if not queryset: filter_kwargs["shared_data"] = True - qs = XForm.objects.filter(**filter_kwargs).only("id", "shared") + queryset = XForm.objects.filter(**filter_kwargs).only("id", "shared") - if not qs: + if not queryset: raise Http404(_("No data matches with given query.")) - return qs + return queryset + # pylint: disable=unused-argument def filter_queryset(self, queryset, view=None): - qs = super(DataViewSet, self).filter_queryset(queryset.only("id", "shared")) - pk = self.kwargs.get(self.lookup_field) + queryset = super().filter_queryset(queryset.only("id", "shared")) + form_pk = self.kwargs.get(self.lookup_field) - if pk: + if form_pk: try: - int(pk) - except ValueError: - if pk == self.public_data_endpoint: - qs = self._get_public_forms_queryset() + int(form_pk) + except ValueError as e: + if form_pk == self.public_data_endpoint: + queryset = self._get_public_forms_queryset() else: - raise ParseError(_("Invalid pk %(pk)s" % {"pk": pk})) + raise ParseError(_(f"Invalid pk {form_pk}")) from e else: - qs = self._filtered_or_shared_qs(qs, pk) + queryset = self._filtered_or_shared_queryset(queryset, form_pk) else: tags = self.request.query_params.get("tags") not_tagged = self.request.query_params.get("not_tagged") if tags and isinstance(tags, six.string_types): tags = tags.split(",") - qs = qs.filter(tags__name__in=tags) + queryset = queryset.filter(tags__name__in=tags) if not_tagged and isinstance(not_tagged, six.string_types): not_tagged = not_tagged.split(",") - qs = qs.exclude(tags__name__in=not_tagged) + queryset = queryset.exclude(tags__name__in=not_tagged) - return qs + return queryset @action( methods=["GET", "POST", "DELETE"], @@ -241,7 +269,11 @@ def filter_queryset(self, queryset, view=None): ], ) def labels(self, request, *args, **kwargs): + """ + Data labels API endpoint. + """ http_status = status.HTTP_400_BAD_REQUEST + # pylint: disable=attribute-defined-outside-init self.object = instance = self.get_object() if request.method == "POST": @@ -278,11 +310,15 @@ def labels(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def enketo(self, request, *args, **kwargs): + """ + Data Enketo URLs endpoint + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() data = {} if isinstance(self.object, XForm): raise ParseError(_("Data id not provided.")) - elif isinstance(self.object, Instance): + if isinstance(self.object, Instance): if request.user.has_perm("change_xform", self.object.xform): return_url = request.query_params.get("return_url") form_url = get_form_url( @@ -304,7 +340,7 @@ def enketo(self, request, *args, **kwargs): if "edit_url" in data: data["url"] = data.pop("edit_url") except EnketoError as e: - raise ParseError(text(e)) + raise ParseError(text(e)) from e else: raise PermissionDenied(_("You do not have edit permissions.")) @@ -315,62 +351,60 @@ def enketo(self, request, *args, **kwargs): def destroy(self, request, *args, **kwargs): instance_ids = request.data.get("instance_ids") delete_all_submissions = strtobool(request.data.get("delete_all", "False")) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if isinstance(self.object, XForm): if not instance_ids and not delete_all_submissions: raise ParseError(_("Data id(s) not provided.")) + initial_count = self.object.submission_count() + if delete_all_submissions: + # Update timestamp only for active records + self.object.instances.filter(deleted_at__isnull=True).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, + ) else: - initial_count = self.object.submission_count() - if delete_all_submissions: - # Update timestamp only for active records - self.object.instances.filter(deleted_at__isnull=True).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user, - ) - else: - instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] - if not instance_ids: - raise ParseError(_("Invalid data ids were provided.")) - - self.object.instances.filter( - id__in=instance_ids, - xform=self.object, - # do not update this timestamp when the record have - # already been deleted. - deleted_at__isnull=True, - ).update( - deleted_at=timezone.now(), - date_modified=timezone.now(), - deleted_by=request.user, - ) - - # updates the num_of_submissions for the form. - after_count = self.object.submission_count(force_update=True) - number_of_records_deleted = initial_count - after_count - - # update the date modified field of the project - self.object.project.date_modified = timezone.now() - self.object.project.save(update_fields=["date_modified"]) - - # send message - send_message( - instance_id=instance_ids, - target_id=self.object.id, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, + instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] + if not instance_ids: + raise ParseError(_("Invalid data ids were provided.")) + + self.object.instances.filter( + id__in=instance_ids, + xform=self.object, + # do not update this timestamp when the record have + # already been deleted. + deleted_at__isnull=True, + ).update( + deleted_at=timezone.now(), + date_modified=timezone.now(), + deleted_by=request.user, ) - return Response( - data={ - "message": "%d records were deleted" % number_of_records_deleted - }, - status=status.HTTP_200_OK, - ) + # updates the num_of_submissions for the form. + after_count = self.object.submission_count(force_update=True) + number_of_records_deleted = initial_count - after_count + + # update the date modified field of the project + self.object.project.date_modified = timezone.now() + self.object.project.save(update_fields=["date_modified"]) + + # send message + send_message( + instance_id=instance_ids, + target_id=self.object.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) - elif isinstance(self.object, Instance): + return Response( + data={"message": f"{number_of_records_deleted} records were deleted"}, + status=status.HTTP_200_OK, + ) + + if isinstance(self.object, Instance): if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): instance_id = self.object.pk @@ -389,30 +423,29 @@ def destroy(self, request, *args, **kwargs): return Response(status=status.HTTP_204_NO_CONTENT) def retrieve(self, request, *args, **kwargs): - data_id, _format = get_data_and_form(kwargs) + _data_id, _format = get_data_and_form(kwargs) + # pylint: disable=attribute-defined-outside-init self.object = instance = self.get_object() if _format == "json" or _format is None or _format == "debug": return Response(instance.json) - elif _format == "xml": + if _format == "xml": return Response(instance.xml) - elif _format == "geojson": - return super(DataViewSet, self).retrieve(request, *args, **kwargs) - elif _format == Attachment.OSM: + if _format == "geojson": + return super().retrieve(request, *args, **kwargs) + if _format == Attachment.OSM: serializer = self.get_serializer(instance.osm_data.all()) return Response(serializer.data) - else: - raise ParseError( - _( - "'%(_format)s' format unknown or not implemented!" - % {"_format": _format} - ) - ) + + raise ParseError(_(f"'{_format}' format unknown or not implemented!")) @action(methods=["GET"], detail=True) def history(self, request, *args, **kwargs): - data_id, _format = get_data_and_form(kwargs) + """ + Return submission history. + """ + _data_id, _format = get_data_and_form(kwargs) instance = self.get_object() # retrieve all history objects and return them @@ -420,14 +453,9 @@ def history(self, request, *args, **kwargs): instance_history = instance.submission_history.all() serializer = InstanceHistorySerializer(instance_history, many=True) return Response(serializer.data) - else: - raise ParseError( - _( - "'%(_format)s' format unknown or not implemented!" - % {"_format": _format} - ) - ) + raise ParseError(_(f"'{_format}' format unknown or not implemented!")) + # pylint: disable=too-many-locals,too-many-branches def _set_pagination_headers( self, xform: XForm, @@ -437,8 +465,6 @@ def _set_pagination_headers( """ Sets the self.headers value for the viewset """ - import math - url = self.request.build_absolute_uri() query = self.request.query_params.get("query") base_url = url.split("?")[0] @@ -475,13 +501,14 @@ def _set_pagination_headers( ) last_page = math.ceil(num_of_records / current_page_size) - if last_page != current_page and last_page != current_page + 1: + if last_page not in (current_page, current_page + 1): last_page_url = f"{base_url}?page={last_page}&page_size={current_page_size}" if current_page != 1: first_page_url = f"{base_url}?page=1&page_size={current_page_size}" if not hasattr(self, "headers"): + # pylint: disable=attribute-defined-outside-init self.headers = {} for rel, link in ( @@ -494,6 +521,7 @@ def _set_pagination_headers( links.append(f'<{link}>; rel="{rel}"') self.headers.update({"Link": ", ".join(links)}) + # pylint: disable=too-many-locals,too-many-branches,too-many-statements def list(self, request, *args, **kwargs): fields = request.GET.get("fields") query = request.GET.get("query", {}) @@ -506,31 +534,34 @@ def list(self, request, *args, **kwargs): is_public_request = lookup == self.public_data_endpoint if lookup_field not in list(kwargs): + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) if is_public_request: + # pylint: disable=attribute-defined-outside-init self.object_list = self._get_public_forms_queryset() elif lookup: - qs = self.filter_queryset(self.get_queryset()).values_list( + queryset = self.filter_queryset(self.get_queryset()).values_list( "pk", "is_merged_dataset" ) - xform_id, is_merged_dataset = qs[0] if qs else (lookup, False) + xform_id, is_merged_dataset = queryset[0] if queryset else (lookup, False) pks = [xform_id] if is_merged_dataset: merged_form = MergedXForm.objects.get(pk=xform_id) - qs = merged_form.xforms.filter(deleted_at__isnull=True).values_list( - "id", "num_of_submissions" - ) + queryset = merged_form.xforms.filter( + deleted_at__isnull=True + ).values_list("id", "num_of_submissions") try: - pks, num_of_submissions = [list(value) for value in zip(*qs)] + pks, num_of_submissions = [list(value) for value in zip(*queryset)] num_of_submissions = sum(num_of_submissions) except ValueError: pks, num_of_submissions = [], 0 else: num_of_submissions = XForm.objects.get(id=xform_id).num_of_submissions + # pylint: disable=attribute-defined-outside-init self.object_list = Instance.objects.filter( xform_id__in=pks, deleted_at=None ).only("json") @@ -538,15 +569,18 @@ def list(self, request, *args, **kwargs): # Enable ordering for XForms with Submissions that are less # than the SUBMISSION_RETRIEVAL_THRESHOLD if num_of_submissions < SUBMISSION_RETRIEVAL_THRESHOLD: + # pylint: disable=attribute-defined-outside-init self.object_list = self.object_list.order_by("id") xform = self.get_object() + # pylint: disable=attribute-defined-outside-init self.object_list = filter_queryset_xform_meta_perms( xform, request.user, self.object_list ) tags = self.request.query_params.get("tags") not_tagged = self.request.query_params.get("not_tagged") + # pylint: disable=attribute-defined-outside-init self.object_list = filters.InstanceFilter( self.request.query_params, queryset=self.object_list, request=request ).qs @@ -578,18 +612,22 @@ def list(self, request, *args, **kwargs): return Response(serializer.data) - elif export_type is None or export_type in ["json"]: + if export_type is None or export_type in ["json"]: # perform default viewset retrieve, no data export - return super(DataViewSet, self).list(request, *args, **kwargs) + return super().list(request, *args, **kwargs) - elif export_type == "geojson": + if export_type == "geojson": serializer = self.get_serializer(self.object_list, many=True) return Response(serializer.data) return custom_response_handler(request, xform, query, export_type) + # pylint: disable=too-many-arguments def set_object_list(self, query, fields, sort, start, limit, is_public_request): + """ + Set the submission instances queryset. + """ try: enable_etag = True if not is_public_request: @@ -599,6 +637,7 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): where, where_params = get_where_clause(query) if where: + # pylint: disable=attribute-defined-outside-init self.object_list = self.object_list.extra( where=where, params=where_params ) @@ -606,15 +645,18 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit + # pylint: disable=attribute-defined-outside-init self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list ) + # pylint: disable=attribute-defined-outside-init self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: query = filter_queryset_xform_meta_perms_sql( self.get_object(), self.request.user, query ) + # pylint: disable=attribute-defined-outside-init self.object_list = query_data( xform, query=query, @@ -625,6 +667,7 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): json_only=not self.kwargs.get("format") == "xml", ) except NoRecordsPermission: + # pylint: disable=attribute-defined-outside-init self.object_list = [] # ETags are Disabled for XForms with Submissions that surpass @@ -649,9 +692,9 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): (get_etag_hash_from_query(records, sql, params)), ) except ValueError as e: - raise ParseError(text(e)) + raise ParseError(text(e)) from e except DataError as e: - raise ParseError(text(e)) + raise ParseError(text(e)) from e def paginate_queryset(self, queryset): if self.paginator is None: @@ -660,6 +703,7 @@ def paginate_queryset(self, queryset): queryset, self.request, view=self, count=self.data_count ) + # pylint: disable=too-many-arguments,too-many-locals def _get_data(self, query, fields, sort, start, limit, is_public_request): self.set_object_list(query, fields, sort, start, limit, is_public_request) @@ -669,7 +713,7 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): self.paginator.page_size_query_param, ] query_param_keys = self.request.query_params - should_paginate = any([k in query_param_keys for k in pagination_keys]) + should_paginate = any(k in query_param_keys for k in pagination_keys) if not should_paginate and not is_public_request: # Paginate requests that try to retrieve data that surpasses @@ -693,12 +737,14 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request): ) try: + # pylint: disable=attribute-defined-outside-init self.object_list = self.paginate_queryset(self.object_list) except OperationalError: + # pylint: disable=attribute-defined-outside-init self.object_list = self.paginate_queryset(self.object_list) - STREAM_DATA = getattr(settings, "STREAM_DATA", False) - if STREAM_DATA: + stream_data = getattr(settings, "STREAM_DATA", False) + if stream_data: response = self._get_streaming_response() else: serializer = self.get_serializer(self.object_list, many=True) @@ -722,6 +768,7 @@ def get_json_string(item): content_type="application/xml", ) else: + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(self.object_list, get_json_string), content_type="application/json", @@ -738,5 +785,10 @@ def get_json_string(item): return response +# pylint: disable=too-many-ancestors class AuthenticatedDataViewSet(DataViewSet): + """ + Authenticated requests only. + """ + permission_classes = (ConnectViewsetPermissions,) diff --git a/onadata/apps/api/viewsets/osm_viewset.py b/onadata/apps/api/viewsets/osm_viewset.py index 55efe858c0..085d636fba 100644 --- a/onadata/apps/api/viewsets/osm_viewset.py +++ b/onadata/apps/api/viewsets/osm_viewset.py @@ -1,145 +1,153 @@ +# -*- coding: utf-8 -*- +""" +The osm API endpoint. +""" from django.http import HttpResponsePermanentRedirect from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ -from rest_framework.viewsets import ReadOnlyModelViewSet from rest_framework.exceptions import ParseError from rest_framework.permissions import AllowAny from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.reverse import reverse +from rest_framework.viewsets import ReadOnlyModelViewSet +from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models import OsmData -from onadata.apps.logger.models.xform import XForm from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance -from onadata.libs.renderers import renderers -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.apps.logger.models.xform import XForm +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.serializers.data_serializer import OSMSerializer -from onadata.libs.serializers.data_serializer import OSMSiteMapSerializer -from onadata.apps.api.tools import get_baseviewset_class +from onadata.libs.renderers import renderers +from onadata.libs.serializers.data_serializer import OSMSerializer, OSMSiteMapSerializer +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() -SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] +SAFE_METHODS = ["GET", "HEAD", "OPTIONS"] -class OsmViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - ReadOnlyModelViewSet): +# pylint: disable=too-many-ancestors +class OsmViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + BaseViewset, + ReadOnlyModelViewSet, +): """ -This endpoint provides public access to OSM submitted data in OSM format. -No authentication is required. Where: + This endpoint provides public access to OSM submitted data in OSM format. + No authentication is required. Where: -* `pk` - the form unique identifier -* `dataid` - submission data unique identifier -* `owner` - username of the owner(user/organization) of the data point + * `pk` - the form unique identifier + * `dataid` - submission data unique identifier + * `owner` - username of the owner(user/organization) of the data point -## GET JSON List of data end points + ## GET JSON List of data end points -Lists the data endpoints accessible to requesting user, for anonymous access -a list of public data endpoints is returned. + Lists the data endpoints accessible to requesting user, for anonymous access + a list of public data endpoints is returned. -
    -GET /api/v1/osm
-
    + 
    +    GET /api/v1/osm
+
    -> Example -> -> curl -X GET https://ona.io/api/v1/osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm -## OSM + ## OSM -The `.osm` file format concatenates all the files for a form or individual - submission. When the `.json` endpoint is accessed, the individual osm files - are listed on the `_attachments` key. + The `.osm` file format concatenates all the files for a form or individual + submission. When the `.json` endpoint is accessed, the individual osm files + are listed on the `_attachments` key. -### OSM endpoint for all osm files uploaded to a form concatenated. + ### OSM endpoint for all osm files uploaded to a form concatenated. -
    -GET /api/v1/osm/{pk}.osm
-
    + 
    +    GET /api/v1/osm/{pk}.osm
+
    -> Example -> -> curl -X GET https://ona.io/api/v1/osm/28058.osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm/28058.osm -### OSM endpoint with all osm files for a specific submission concatenated. + ### OSM endpoint with all osm files for a specific submission concatenated. -
    -GET /api/v1/osm/{pk}/{data_id}.osm
-
    + 
    +    GET /api/v1/osm/{pk}/{data_id}.osm
+
    -> Example -> -> curl -X GET https://ona.io/api/v1/osm/28058/20.osm + > Example + > + > curl -X GET https://ona.io/api/v1/osm/28058/20.osm + """ -""" renderer_classes = [ renderers.OSMRenderer, JSONRenderer, ] serializer_class = OSMSerializer - permission_classes = (AllowAny, ) - lookup_field = 'pk' - lookup_fields = ('pk', 'dataid') + permission_classes = (AllowAny,) + lookup_field = "pk" + lookup_fields = ("pk", "dataid") extra_lookup_fields = None - public_data_endpoint = 'public' + public_data_endpoint = "public" queryset = XForm.objects.filter().select_related() def get_serializer_class(self): - pk = self.kwargs.get('pk') - if self.action == 'list' and pk is None: + form_pk = self.kwargs.get("pk") + if self.action == "list" and form_pk is None: return OSMSiteMapSerializer - return super(OsmViewSet, self).get_serializer_class() + return super().get_serializer_class() def filter_queryset(self, queryset): - pk = self.kwargs.get('pk') - if pk: - queryset = queryset.filter(pk=pk) + form_pk = self.kwargs.get("pk") + if form_pk: + queryset = queryset.filter(pk=form_pk) - return super(OsmViewSet, self).filter_queryset(queryset) + return super().filter_queryset(queryset) def get_object(self): - obj = super(OsmViewSet, self).get_object() + obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - if pk is not None and dataid is not None: + if form_pk is not None and dataid is not None: try: int(dataid) - except ValueError: - raise ParseError(_(u"Invalid dataid %(dataid)s" - % {'dataid': dataid})) + except ValueError as e: + raise ParseError(_(f"Invalid dataid {dataid}")) from e - obj = get_object_or_404(Instance, pk=dataid, xform__pk=pk) + obj = get_object_or_404(Instance, pk=dataid, xform__pk=form_pk) return obj def retrieve(self, request, *args, **kwargs): - fmt = kwargs.get('format', request.accepted_renderer.format) - if fmt != 'osm': + fmt = kwargs.get("format", request.accepted_renderer.format) + if fmt != "osm": pk_lookup, dataid_lookup = self.lookup_fields - pk = self.kwargs.get(pk_lookup) + form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) - kwargs = {'pk': pk, 'format': 'osm'} - viewname = 'osm-list' + kwargs = {"pk": form_pk, "format": "osm"} + viewname = "osm-list" if dataid: kwargs[dataid_lookup] = dataid - viewname = 'osm-detail' + viewname = "osm-detail" return HttpResponsePermanentRedirect( - reverse(viewname, kwargs=kwargs, request=request)) + reverse(viewname, kwargs=kwargs, request=request) + ) instance = self.get_object() if isinstance(instance, XForm): @@ -151,13 +159,17 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) def list(self, request, *args, **kwargs): - fmt = kwargs.get('format', request.accepted_renderer.format) - pk = kwargs.get('pk') - if pk: - if fmt != 'osm': + fmt = kwargs.get("format", request.accepted_renderer.format) + form_pk = kwargs.get("pk") + if form_pk: + if fmt != "osm": return HttpResponsePermanentRedirect( - reverse('osm-list', kwargs={'pk': pk, 'format': 'osm'}, - request=request)) + reverse( + "osm-list", + kwargs={"pk": form_pk, "format": "osm"}, + request=request, + ) + ) instance = self.filter_queryset(self.get_queryset()) osm_list = OsmData.objects.filter(instance__xform__in=instance) page = self.paginate_queryset(osm_list) @@ -168,15 +180,21 @@ def list(self, request, *args, **kwargs): return Response(serializer.data) - if fmt == 'osm': + if fmt == "osm": return HttpResponsePermanentRedirect( - reverse('osm-list', kwargs={'format': 'json'}, - request=request)) - instances = Attachment.objects.filter(extension='osm').values( - 'instance__xform', 'instance__xform__user__username', - 'instance__xform__title', 'instance__xform__id_string')\ - .order_by('instance__xform__id')\ - .distinct('instance__xform__id') + reverse("osm-list", kwargs={"format": "json"}, request=request) + ) + instances = ( + Attachment.objects.filter(extension="osm") + .values( + "instance__xform", + "instance__xform__user__username", + "instance__xform__title", + "instance__xform__id_string", + ) + .order_by("instance__xform__id") + .distinct("instance__xform__id") + ) serializer = self.get_serializer(instances, many=True) - return Response(serializer.data, content_type='application/json') + return Response(serializer.data, content_type="application/json") diff --git a/onadata/apps/logger/models/osmdata.py b/onadata/apps/logger/models/osmdata.py index d79372fdaf..91013ec62d 100644 --- a/onadata/apps/logger/models/osmdata.py +++ b/onadata/apps/logger/models/osmdata.py @@ -2,7 +2,6 @@ """ OSM Data model class """ -from django.db.models import JSONField from django.contrib.gis.db import models @@ -17,7 +16,7 @@ class OsmData(models.Model): xml = models.TextField() osm_id = models.CharField(max_length=20) osm_type = models.CharField(max_length=10, default="way") - tags = JSONField(default=dict, null=False) + tags = models.JSONField(default=dict, null=False) geom = models.GeometryCollectionField() filename = models.CharField(max_length=255) field_name = models.CharField(max_length=255, blank=True, default="") @@ -44,6 +43,9 @@ def get_tag_keys(cls, xform, field_path, include_prefix=False): return sorted([prefix + key.id for key in query]) def get_tags_with_prefix(self): + """ + Returns tags prefixed by the field_name. + """ doc = {self.field_name + ":" + self.osm_type + ":id": self.osm_id} for k, v in self.tags.items(): doc[self.field_name + ":" + k] = v @@ -61,6 +63,10 @@ def _set_centroid_in_tags(self): } ) - def save(self, *args, **kwargs): + def save( + self, force_insert=False, force_update=False, using=None, update_fields=None + ): self._set_centroid_in_tags() - super().save(*args, **kwargs) + super().save( + force_insert=False, force_update=False, using=None, update_fields=None + ) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 1bbf05f569..66d8207ef7 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -171,9 +171,9 @@ def _start_index_limit(records, sql, fields, params, sort, start_index, limit): has_json_fields = ParsedInstance._has_json_fields(sort) if start_index is not None and (has_json_fields or fields): params += [start_index] - sql = f"{sql} OFFSET %%s" + sql = f"{sql} OFFSET %s" if limit is not None and (has_json_fields or fields): - sql = f"{sql} LIMIT %%s" + sql = f"{sql} LIMIT %s" params += [limit] if ( start_index is not None From a519c203e9275661c93370ddf72bc8a99ad2da1d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 08:49:48 +0300 Subject: [PATCH 085/234] batch: cleanup --- onadata/apps/api/viewsets/data_viewset.py | 18 +- onadata/apps/api/viewsets/project_viewset.py | 169 ++++++++++--------- onadata/apps/api/viewsets/team_viewset.py | 77 +++++---- onadata/apps/main/models/audit.py | 13 +- onadata/libs/data/__init__.py | 17 ++ 5 files changed, 167 insertions(+), 127 deletions(-) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 6c05e76a5d..74ba698d9a 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -41,7 +41,7 @@ query_data, ) from onadata.libs import filters -from onadata.libs.data import parse_int +from onadata.libs.data import parse_int, strtobool from onadata.libs.exceptions import EnketoError, NoRecordsPermission from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( AnonymousUserPublicFormsMixin, @@ -77,22 +77,6 @@ BaseViewset = get_baseviewset_class() -# source from deprecated module distutils/util.py -def strtobool(val): - """Convert a string representation of truth to true (1) or false (0). - - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - """ - val = val.lower() - if val in ("y", "yes", "t", "true", "on", "1"): - return 1 - if val in ("n", "no", "f", "false", "off", "0"): - return 0 - raise ValueError(f"invalid truth value {val}") - - def get_data_and_form(kwargs): """ Checks if the dataid in ``kwargs`` is a valid integer. diff --git a/onadata/apps/api/viewsets/project_viewset.py b/onadata/apps/api/viewsets/project_viewset.py index c21b07eecc..4439b793ca 100644 --- a/onadata/apps/api/viewsets/project_viewset.py +++ b/onadata/apps/api/viewsets/project_viewset.py @@ -1,8 +1,10 @@ -from distutils.util import strtobool - +# -*- coding=utf-8 -*- +""" +The /projects API endpoint implementation. +""" +from django.core.cache import cache from django.core.mail import send_mail from django.shortcuts import get_object_or_404 -from django.core.cache import cache from rest_framework import status from rest_framework.decorators import action @@ -15,124 +17,128 @@ from onadata.apps.logger.models import Project, XForm from onadata.apps.main.models import UserProfile from onadata.apps.main.models.meta_data import MetaData -from onadata.libs.filters import (AnonUserProjectFilter, ProjectOwnerFilter, - TagFilter) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.data import strtobool +from onadata.libs.filters import AnonUserProjectFilter, ProjectOwnerFilter, TagFilter +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.labels_mixin import LabelsMixin from onadata.libs.mixins.profiler_mixin import ProfilerMixin -from onadata.libs.serializers.project_serializer import \ - (BaseProjectSerializer, ProjectSerializer) -from onadata.libs.serializers.share_project_serializer import \ - (RemoveUserFromProjectSerializer, ShareProjectSerializer) -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileSerializer -from onadata.libs.utils.cache_tools import ( - PROJ_OWNER_CACHE, safe_delete) -from onadata.libs.serializers.xform_serializer import (XFormCreateSerializer, - XFormSerializer) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) +from onadata.libs.serializers.share_project_serializer import ( + RemoveUserFromProjectSerializer, + ShareProjectSerializer, +) +from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer +from onadata.libs.serializers.xform_serializer import ( + XFormCreateSerializer, + XFormSerializer, +) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete from onadata.libs.utils.common_tools import merge_dicts from onadata.libs.utils.export_tools import str_to_bool from onadata.settings.common import DEFAULT_FROM_EMAIL, SHARE_PROJECT_SUBJECT +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() -class ProjectViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, LabelsMixin, ProfilerMixin, - BaseViewset, ModelViewSet): +# pylint: disable=too-many-ancestors +class ProjectViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + LabelsMixin, + ProfilerMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create Project and Project Forms. """ + queryset = Project.objects.filter(deleted_at__isnull=True).select_related() serializer_class = ProjectSerializer - lookup_field = 'pk' + lookup_field = "pk" extra_lookup_fields = None permission_classes = [ProjectPermissions] - filter_backends = (AnonUserProjectFilter, - ProjectOwnerFilter, - TagFilter) + filter_backends = (AnonUserProjectFilter, ProjectOwnerFilter, TagFilter) def get_serializer_class(self): - action = self.action - - if action == "list": - serializer_class = BaseProjectSerializer - else: - serializer_class = \ - super(ProjectViewSet, self).get_serializer_class() - - return serializer_class + if self.action == "list": + return BaseProjectSerializer + return super().get_serializer_class() def get_queryset(self): - if self.request.method.upper() in ['GET', 'OPTIONS']: + if self.request.method.upper() in ["GET", "OPTIONS"]: self.queryset = Project.prefetched.filter( - deleted_at__isnull=True, organization__is_active=True) + deleted_at__isnull=True, organization__is_active=True + ) - return super(ProjectViewSet, self).get_queryset() + return super().get_queryset() def update(self, request, *args, **kwargs): - project_id = kwargs.get('pk') - response = super(ProjectViewSet, self).update(request, *args, **kwargs) - cache.set(f'{PROJ_OWNER_CACHE}{project_id}', response.data) + project_id = kwargs.get("pk") + response = super().update(request, *args, **kwargs) + cache.set(f"{PROJ_OWNER_CACHE}{project_id}", response.data) return response def retrieve(self, request, *args, **kwargs): - """ Retrieve single project """ - project_id = kwargs.get('pk') - project = cache.get(f'{PROJ_OWNER_CACHE}{project_id}') + """Retrieve single project""" + project_id = kwargs.get("pk") + project = cache.get(f"{PROJ_OWNER_CACHE}{project_id}") if project: return Response(project) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - serializer = ProjectSerializer( - self.object, context={'request': request}) + serializer = ProjectSerializer(self.object, context={"request": request}) return Response(serializer.data) - @action(methods=['POST', 'GET'], detail=True) + @action(methods=["POST", "GET"], detail=True) def forms(self, request, **kwargs): """Add a form to a project or list forms for the project. The request key `xls_file` holds the XLSForm file object. """ + # pylint: disable=attribute-defined-outside-init project = self.object = self.get_object() - if request.method.upper() == 'POST': + if request.method.upper() == "POST": survey = utils.publish_project_xform(request, project) if isinstance(survey, XForm): - if 'formid' in request.data: + if "formid" in request.data: serializer_cls = XFormSerializer else: serializer_cls = XFormCreateSerializer - serializer = serializer_cls(survey, - context={'request': request}) + serializer = serializer_cls(survey, context={"request": request}) - published_by_formbuilder = request.data.get( - 'published_by_formbuilder' - ) + published_by_formbuilder = request.data.get("published_by_formbuilder") if str_to_bool(published_by_formbuilder): - MetaData.published_by_formbuilder(survey, 'True') + MetaData.published_by_formbuilder(survey, "True") - return Response(serializer.data, - status=status.HTTP_201_CREATED) + return Response(serializer.data, status=status.HTTP_201_CREATED) return Response(survey, status=status.HTTP_400_BAD_REQUEST) xforms = XForm.objects.filter(project=project) - serializer = XFormSerializer(xforms, context={'request': request}, - many=True) + serializer = XFormSerializer(xforms, context={"request": request}, many=True) return Response(serializer.data) - @action(methods=['PUT'], detail=True) + @action(methods=["PUT"], detail=True) def share(self, request, *args, **kwargs): + """ + Allow sharing of a project to a user. + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - data = merge_dicts(request.data.dict(), {'project': self.object.pk}) + data = merge_dicts(request.data.dict(), {"project": self.object.pk}) remove = data.get("remove") if remove and remove is not isinstance(remove, bool): @@ -144,7 +150,7 @@ def share(self, request, *args, **kwargs): serializer = ShareProjectSerializer(data=data) if serializer.is_valid(): serializer.save() - email_msg = data.get('email_msg') + email_msg = data.get("email_msg") if email_msg: # send out email message. try: @@ -154,37 +160,46 @@ def share(self, request, *args, **kwargs): user = instance.user send_mail( SHARE_PROJECT_SUBJECT.format(self.object.name), - email_msg, DEFAULT_FROM_EMAIL, (user.email, )) + email_msg, + DEFAULT_FROM_EMAIL, + (user.email,), + ) else: - send_mail(SHARE_PROJECT_SUBJECT.format(self.object.name), - email_msg, DEFAULT_FROM_EMAIL, (user.email,)) + send_mail( + SHARE_PROJECT_SUBJECT.format(self.object.name), + email_msg, + DEFAULT_FROM_EMAIL, + (user.email,), + ) else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) # clear cache - safe_delete(f'{PROJ_OWNER_CACHE}{self.object.pk}') + safe_delete(f"{PROJ_OWNER_CACHE}{self.object.pk}") return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['DELETE', 'GET', 'POST'], detail=True) + @action(methods=["DELETE", "GET", "POST"], detail=True) def star(self, request, *args, **kwargs): + """ + Allows to add a user that stars a project. + """ user = request.user - self.object = project = get_object_or_404(Project, - pk=kwargs.get('pk')) + # pylint: disable=attribute-defined-outside-init + self.object = project = get_object_or_404(Project, pk=kwargs.get("pk")) - if request.method == 'DELETE': + if request.method == "DELETE": project.user_stars.remove(user) project.save() - elif request.method == 'POST': + elif request.method == "POST": project.user_stars.add(user) project.save() - elif request.method == 'GET': - users = project.user_stars.values('pk') + elif request.method == "GET": + users = project.user_stars.values("pk") user_profiles = UserProfile.objects.filter(user__in=users) - serializer = UserProfileSerializer(user_profiles, - context={'request': request}, - many=True) + serializer = UserProfileSerializer( + user_profiles, context={"request": request}, many=True + ) return Response(serializer.data) diff --git a/onadata/apps/api/viewsets/team_viewset.py b/onadata/apps/api/viewsets/team_viewset.py index 4a9bae83d7..aacad71502 100644 --- a/onadata/apps/api/viewsets/team_viewset.py +++ b/onadata/apps/api/viewsets/team_viewset.py @@ -1,83 +1,99 @@ -from distutils.util import strtobool - -from django.contrib.auth.models import User +# -*- coding=utf-8 -*- +""" +The /teams API endpoint implementation. +""" +from django.contrib.auth import get_user_model from django.utils.translation import ugettext as _ from rest_framework import status -from rest_framework_guardian.filters import ObjectPermissionsFilter from rest_framework.decorators import action from rest_framework.permissions import DjangoObjectPermissions from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from rest_framework_guardian.filters import ObjectPermissionsFilter from onadata.apps.api.models import Team -from onadata.apps.api.tools import (add_user_to_team, get_baseviewset_class, - remove_user_from_team) +from onadata.apps.api.tools import ( + add_user_to_team, + get_baseviewset_class, + remove_user_from_team, +) +from onadata.libs.data import strtobool from onadata.libs.filters import TeamOrgFilter -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.serializers.share_team_project_serializer import \ - (RemoveTeamFromProjectSerializer, ShareTeamProjectSerializer) +from onadata.libs.serializers.share_team_project_serializer import ( + RemoveTeamFromProjectSerializer, + ShareTeamProjectSerializer, +) from onadata.libs.serializers.team_serializer import TeamSerializer from onadata.libs.utils.common_tools import merge_dicts +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() +# pylint: disable=invalid-name +User = get_user_model() -class TeamViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, - BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class TeamViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): """ This endpoint allows you to create, update and view team information. """ + queryset = Team.objects.all() serializer_class = TeamSerializer - lookup_field = 'pk' + lookup_field = "pk" extra_lookup_fields = None permission_classes = [DjangoObjectPermissions] - filter_backends = (ObjectPermissionsFilter, - TeamOrgFilter) + filter_backends = (ObjectPermissionsFilter, TeamOrgFilter) - @action(methods=['DELETE', 'GET', 'POST'], detail=True) + @action(methods=["DELETE", "GET", "POST"], detail=True) def members(self, request, *args, **kwargs): + """ + Returns members of an organization. + """ team = self.get_object() data = {} status_code = status.HTTP_200_OK - if request.method in ['DELETE', 'POST']: - username = request.data.get('username') or\ - request.query_params.get('username') + if request.method in ["DELETE", "POST"]: + username = request.data.get("username") or request.query_params.get( + "username" + ) if username: try: user = User.objects.get(username__iexact=username) except User.DoesNotExist: status_code = status.HTTP_400_BAD_REQUEST - data['username'] = [ - _(u"User `%(username)s` does not exist." - % {'username': username})] + data["username"] = [_(f"User `{username}` does not exist.")] else: - if request.method == 'POST': + if request.method == "POST": add_user_to_team(team, user) - elif request.method == 'DELETE': + elif request.method == "DELETE": remove_user_from_team(team, user) status_code = status.HTTP_201_CREATED else: status_code = status.HTTP_400_BAD_REQUEST - data['username'] = [_(u"This field is required.")] + data["username"] = [_("This field is required.")] if status_code in [status.HTTP_200_OK, status.HTTP_201_CREATED]: data = [u.username for u in team.user_set.all()] return Response(data, status=status_code) - @action(methods=['POST'], detail=True) + @action(methods=["POST"], detail=True) def share(self, request, *args, **kwargs): + """ + Performs sharing a team project operations. + """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - data = merge_dicts(request.data.items(), {'team': self.object.pk}) + data = merge_dicts(request.data.items(), {"team": self.object.pk}) remove = data.get("remove") if remove and remove is not isinstance(remove, bool): @@ -91,7 +107,6 @@ def share(self, request, *args, **kwargs): if serializer.is_valid(): serializer.save() else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 208c0192ce..27237aed3a 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -7,14 +7,17 @@ from django.db import models from django.db import connection -from django.db.models import JSONField from django.utils.translation import ugettext as _ DEFAULT_LIMIT = 1000 class Audit(models.Model): - json = JSONField() + """ + Audit model - persists audit logs. + """ + + json = models.JSONField() class Meta: app_label = "main" @@ -33,6 +36,9 @@ def __init__(self, data): self.data = data def save(self): + """ + Persists an audit to the DB + """ audit = Audit(json=self.data) audit.save() @@ -88,6 +94,9 @@ def query_data( limit=DEFAULT_LIMIT, count=False, ): + """ + Queries the Audit model and returns an iterator of the records. + """ if start is not None and (start < 0 or limit < 0): raise ValueError(_("Invalid start/limit params")) diff --git a/onadata/libs/data/__init__.py b/onadata/libs/data/__init__.py index aec01f3cd4..a8efc0e3a5 100644 --- a/onadata/libs/data/__init__.py +++ b/onadata/libs/data/__init__.py @@ -16,3 +16,20 @@ def parse_int(num): return num and int(num) except ValueError: pass + return None + + +# source from deprecated module distutils/util.py +def strtobool(val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return 1 + if val in ("n", "no", "f", "false", "off", "0"): + return 0 + raise ValueError(f"invalid truth value {val}") From 41fbfac09bb40d57fc8a5bdb2611b7d1ae2bf2c4 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 08:56:05 +0300 Subject: [PATCH 086/234] Fix missing f-string format --- .../tests/viewsets/test_connect_viewset.py | 476 +++++++++--------- onadata/libs/authentication.py | 2 +- 2 files changed, 238 insertions(+), 240 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_connect_viewset.py b/onadata/apps/api/tests/viewsets/test_connect_viewset.py index 9b23639510..0ede55e361 100644 --- a/onadata/apps/api/tests/viewsets/test_connect_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_connect_viewset.py @@ -1,3 +1,7 @@ +# -*- coding=utf-8 -*- +""" +Test /user API endpoint +""" from datetime import datetime from datetime import timedelta @@ -16,11 +20,9 @@ from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.models.odk_token import ODKToken -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet -from onadata.libs.serializers.password_reset_serializer import \ - default_token_generator +from onadata.libs.serializers.password_reset_serializer import default_token_generator from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.libs.authentication import DigestAuthentication from onadata.libs.serializers.project_serializer import ProjectSerializer @@ -28,43 +30,48 @@ class TestConnectViewSet(TestAbstractViewSet): - def setUp(self): super(self.__class__, self).setUp() - self.view = ConnectViewSet.as_view({ - "get": "list", - "post": "reset", - "delete": "expire", - }) + self.view = ConnectViewSet.as_view( + { + "get": "list", + "post": "reset", + "delete": "expire", + } + ) self.data = { - 'url': 'http://testserver/api/v1/profiles/bob', - 'username': u'bob', - 'name': u'Bob', - 'email': u'bob@columbia.edu', - 'city': u'Bobville', - 'country': u'US', - 'organization': u'Bob Inc.', - 'website': u'bob.com', - 'twitter': u'boberama', - 'gravatar': self.user.profile.gravatar, - 'require_auth': False, - 'user': 'http://testserver/api/v1/users/bob', - 'api_token': self.user.auth_token.key, + "url": "http://testserver/api/v1/profiles/bob", + "username": "bob", + "name": "Bob", + "email": "bob@columbia.edu", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "website": "bob.com", + "twitter": "boberama", + "gravatar": self.user.profile.gravatar, + "require_auth": False, + "user": "http://testserver/api/v1/users/bob", + "api_token": self.user.auth_token.key, } def test_generate_auth_token(self): - self.view = ConnectViewSet.as_view({ - "post": "create", - }) + self.view = ConnectViewSet.as_view( + { + "post": "create", + } + ) request = self.factory.post("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 201) def test_regenerate_auth_token(self): - self.view = ConnectViewSet.as_view({ - "get": "regenerate_auth_token", - }) + self.view = ConnectViewSet.as_view( + { + "get": "regenerate_auth_token", + } + ) prev_token = self.user.auth_token request = self.factory.get("/", **self.extra) response = self.view(request) @@ -72,62 +79,64 @@ def test_regenerate_auth_token(self): new_token = Token.objects.get(user=self.user) self.assertNotEqual(prev_token, new_token) - self.view = ConnectViewSet.as_view({ - "get": "list", - }) - self.extra = {'HTTP_AUTHORIZATION': 'Token %s' % new_token} - request = self.factory.get('/', **self.extra) + self.view = ConnectViewSet.as_view( + { + "get": "list", + } + ) + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % new_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 200) - self.extra = {'HTTP_AUTHORIZATION': 'Token invalidtoken'} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "Token invalidtoken"} + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response['www-authenticate'], "Token") + self.assertEqual(response["www-authenticate"], "Token") def test_get_profile(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(dict(response.data), self.data) def test_using_valid_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(temp_token, response.data['temp_token']) + self.assertEqual(temp_token, response.data["temp_token"]) def test_using_invalid_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = 'abcdefghijklmopqrstuvwxyz' + temp_token = "abcdefghijklmopqrstuvwxyz" - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], 'Invalid token') - self.assertEqual(response['www-authenticate'], "TempToken") + self.assertEqual(response.data["detail"], "Invalid token") + self.assertEqual(response["www-authenticate"], "TempToken") def test_using_expired_temp_token(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] temp_token_obj = TempToken.objects.get(key=temp_token) day = timedelta(seconds=settings.DEFAULT_TEMP_TOKEN_EXPIRY_TIME) @@ -136,67 +145,65 @@ def test_using_expired_temp_token(self): temp_token_obj.created = yesterday temp_token_obj.save() - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % - temp_token_obj.key} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token_obj.key} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - self.assertEqual(response.data['detail'], 'Token expired') + self.assertEqual(response.data["detail"], "Token expired") def test_expire_temp_token_using_expire_endpoint(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) - temp_token = response.data['temp_token'] + temp_token = response.data["temp_token"] # expire temporary token - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 204) # try to expire temporary token for the second time - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['detail'], 'Temporary token not found!') + self.assertEqual(response.data["detail"], "Temporary token not found!") # try to login with deleted temporary token - self.extra = {'HTTP_AUTHORIZATION': 'TempToken %s' % temp_token} - request = self.factory.get('/', **self.extra) + self.extra = {"HTTP_AUTHORIZATION": "TempToken %s" % temp_token} + request = self.factory.get("/", **self.extra) request.session = self.client.session response = self.view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - 'Invalid token') - self.assertEqual(response['www-authenticate'], "TempToken") + self.assertEqual(response.data["detail"], "Invalid token") + self.assertEqual(response["www-authenticate"], "TempToken") def test_get_starred_projects(self): self._project_create() # add star as bob - view = ProjectViewSet.as_view({ - 'get': 'star', - 'post': 'star' - }) - request = self.factory.post('/', **self.extra) + view = ProjectViewSet.as_view({"get": "star", "post": "star"}) + request = self.factory.post("/", **self.extra) response = view(request, pk=self.project.pk) # get starred projects - view = ConnectViewSet.as_view({ - 'get': 'starred', - }) - request = self.factory.get('/', **self.extra) + view = ConnectViewSet.as_view( + { + "get": "starred", + } + ) + request = self.factory.get("/", **self.extra) response = view(request, user=self.user) self.assertEqual(response.status_code, 200) self.project.refresh_from_db() request.user = self.user self.project_data = ProjectSerializer( - self.project, context={'request': request}).data - del self.project_data['date_modified'] - del response.data[0]['date_modified'] + self.project, context={"request": request} + ).data + del self.project_data["date_modified"] + del response.data[0]["date_modified"] self.assertEqual(len(response.data), 1) self.assertDictEqual(dict(response.data[0]), dict(self.project_data)) @@ -205,81 +212,85 @@ def test_user_list_with_digest(self): cache.clear() view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) - request = self.factory.head('/') + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) + request = self.factory.head("/") - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") response = view(request) - self.assertTrue(response.has_header('WWW-Authenticate')) - self.assertTrue(response['WWW-Authenticate'].startswith('Digest ')) - self.assertIn('nonce=', response['WWW-Authenticate']) - request = self.factory.get('/') + self.assertTrue(response.has_header("WWW-Authenticate")) + self.assertTrue(response["WWW-Authenticate"].startswith("Digest ")) + self.assertIn("nonce=", response["WWW-Authenticate"]) + request = self.factory.get("/") request.META.update(auth(request.META, response)) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll " - u"have to wait 30 minutes before trying again.") - auth = DigestAuth('bob', 'bobbob') + self.assertEqual( + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll " + "have to wait 30 minutes before trying again.", + ) + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) request.session = self.client.session response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) def test_user_list_with_basic_and_digest(self): view = ConnectViewSet.as_view( - {'get': 'list'}, + {"get": "list"}, authentication_classes=( DigestAuthentication, - authentication.BasicAuthentication - )) - request = self.factory.get('/') - auth = BasicAuth('bob', 'bob') + authentication.BasicAuthentication, + ), + ) + request = self.factory.get("/") + auth = BasicAuth("bob", "bob") request.META.update(auth(request.META)) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password.") - auth = BasicAuth('bob', 'bobbob') + self.assertEqual(response.data["detail"], "Invalid username/password.") + auth = BasicAuth("bob", "bobbob") request.META.update(auth(request.META)) request.session = self.client.session response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) - @patch('onadata.libs.serializers.password_reset_serializer.send_mail') + @patch("onadata.libs.serializers.password_reset_serializer.send_mail") def test_request_reset_password(self, mock_send_mail): - data = {'email': self.user.email, - 'reset_url': u'http://testdomain.com/reset_form'} - request = self.factory.post('/', data=data) + data = { + "email": self.user.email, + "reset_url": "http://testdomain.com/reset_form", + } + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 204, response.data) self.assertTrue(mock_send_mail.called) - data['email_subject'] = 'X' * 100 - request = self.factory.post('/', data=data) + data["email_subject"] = "X" * 100 + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['email_subject'][0], - u'Ensure this field has no more than 78 characters.' + response.data["email_subject"][0], + "Ensure this field has no more than 78 characters.", ) mock_send_mail.called = False - request = self.factory.post('/') + request = self.factory.post("/") response = self.view(request) self.assertFalse(mock_send_mail.called) self.assertEqual(response.status_code, 400) @@ -291,23 +302,22 @@ def test_reset_user_password(self): self.user.save() token = default_token_generator.make_token(self.user) new_password = "bobbob1" - data = {'token': token, 'new_password': new_password} + data = {"token": token, "new_password": new_password} # missing uid, should fail - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) - data['uid'] = urlsafe_base64_encode( - force_bytes(self.user.pk)) + data["uid"] = urlsafe_base64_encode(force_bytes(self.user.pk)) # with uid, should be successful - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 200) user = User.objects.get(email=self.user.email) - self.assertEqual(user.username, response.data['username']) + self.assertEqual(user.username, response.data["username"]) self.assertTrue(user.check_password(new_password)) - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) @@ -317,12 +327,10 @@ def test_reset_user_password_with_updated_user_email(self): self.user.last_login = now() self.user.save() new_password = "bobbob1" - uid = urlsafe_base64_encode( - force_bytes(self.user.pk)) + uid = urlsafe_base64_encode(force_bytes(self.user.pk)) mhv = default_token_generator token = mhv.make_token(self.user) - data = {'token': token, 'new_password': new_password, - 'uid': uid} + data = {"token": token, "new_password": new_password, "uid": uid} # check that the token is valid valid_token = mhv.check_token(self.user, token) self.assertTrue(valid_token) @@ -336,18 +344,19 @@ def test_reset_user_password_with_updated_user_email(self): invalid_token = mhv.check_token(self.user, token) self.assertFalse(invalid_token) - request = self.factory.post('/', data=data) + request = self.factory.post("/", data=data) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertTrue( - 'Invalid token' in response.data['non_field_errors'][0]) + self.assertTrue("Invalid token" in response.data["non_field_errors"][0]) - @patch('onadata.libs.serializers.password_reset_serializer.send_mail') + @patch("onadata.libs.serializers.password_reset_serializer.send_mail") def test_request_reset_password_custom_email_subject(self, mock_send_mail): - data = {'email': self.user.email, - 'reset_url': u'http://testdomain.com/reset_form', - 'email_subject': 'You requested for a reset password'} - request = self.factory.post('/', data=data) + data = { + "email": self.user.email, + "reset_url": "http://testdomain.com/reset_form", + "email_subject": "You requested for a reset password", + } + request = self.factory.post("/", data=data) response = self.view(request) self.assertTrue(mock_send_mail.called) @@ -357,30 +366,32 @@ def test_user_updates_email_wrong_password(self): # Clear cache cache.clear() view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bob') + auth = DigestAuth("bob@columbia.edu", "bob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") + self.assertEqual( + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) def test_user_updates_email(self): view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('bob@columbia.edu', 'bobbob') + auth = DigestAuth("bob@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) @@ -388,35 +399,35 @@ def test_user_updates_email(self): self.user.save() update_partial_digests(self.user, "bobbob") - auth = DigestAuth('bob2@columbia.edu', 'bobbob') + auth = DigestAuth("bob2@columbia.edu", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) - temp_token = TempToken.objects.get(user__username='bob') - self.data['temp_token'] = temp_token.key - self.data['email'] = 'bob2@columbia.edu' + temp_token = TempToken.objects.get(user__username="bob") + self.data["temp_token"] = temp_token.key + self.data["email"] = "bob2@columbia.edu" self.assertEqual(response.status_code, 200) def test_user_has_no_profile_bug(self): - alice = User.objects.create(username='alice') - alice.set_password('alice') + alice = User.objects.create(username="alice") + alice.set_password("alice") update_partial_digests(alice, "alice") view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - @patch('onadata.apps.api.tasks.send_account_lockout_email.apply_async') + @patch("onadata.apps.api.tasks.send_account_lockout_email.apply_async") def test_login_attempts(self, send_account_lockout_email): view = ConnectViewSet.as_view( - {'get': 'list'}, - authentication_classes=(DigestAuthentication,)) - auth = DigestAuth('bob', 'bob') + {"get": "list"}, authentication_classes=(DigestAuthentication,) + ) + auth = DigestAuth("bob", "bob") # clear cache cache.clear() @@ -425,90 +436,88 @@ def test_login_attempts(self, send_account_lockout_email): # first time it creates a cache response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 9 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") - request_ip = request.META.get('REMOTE_ADDR') self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 1) + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 9 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) + request_ip = request.META.get("REMOTE_ADDR") + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 1) # cache value increments with subsequent attempts response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Invalid username/password. For security reasons, " - u"after 8 more failed login attempts you'll have to " - u"wait 30 minutes before trying again.") self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) + response.data["detail"], + "Invalid username/password. For security reasons, " + "after 8 more failed login attempts you'll have to " + "wait 30 minutes before trying again.", + ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) # login attempts are tracked separately for other IPs - request.META.update({'HTTP_X_REAL_IP': '5.6.7.8'}) + request.META.update({"HTTP_X_REAL_IP": "5.6.7.8"}) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) - self.assertEqual( - cache.get(safe_key('login_attempts-5.6.7.8-bob')), 1 - ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) + self.assertEqual(cache.get(safe_key("login_attempts-5.6.7.8-bob")), 1) # login_attempts doesn't increase with correct login - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 2) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) # lockout_user cache created upon fifth attempt - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") request = self._get_request_session_with_auth(view, auth) self.assertFalse(send_account_lockout_email.called) - cache.set(safe_key(f'login_attempts-{request_ip}-bob'), 9) - self.assertIsNone(cache.get(safe_key(f'lockout_ip-{request_ip}-bob'))) + cache.set(safe_key(f"login_attempts-{request_ip}-bob"), 9) + self.assertIsNone(cache.get(safe_key(f"lockout_ip-{request_ip}-bob"))) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Locked out. Too many wrong username/password " - u"attempts. Try again in 30 minutes.") self.assertEqual( - cache.get(safe_key(f'login_attempts-{request_ip}-bob')), 10) - self.assertIsNotNone( - cache.get(safe_key(f'lockout_ip-{request_ip}-bob'))) + response.data["detail"], + "Locked out. Too many wrong username/password " + "attempts. Try again in 30 minutes.", + ) + self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 10) + self.assertIsNotNone(cache.get(safe_key(f"lockout_ip-{request_ip}-bob"))) lockout = datetime.strptime( - cache.get(safe_key(f'lockout_ip-{request_ip}-bob')), - '%Y-%m-%dT%H:%M:%S') + cache.get(safe_key(f"lockout_ip-{request_ip}-bob")), "%Y-%m-%dT%H:%M:%S" + ) self.assertIsInstance(lockout, datetime) # email sent upon limit being reached with right arguments - subject_path = 'account_lockout/lockout_email_subject.txt' + subject_path = "account_lockout/lockout_email_subject.txt" self.assertTrue(send_account_lockout_email.called) email_subject = render_to_string(subject_path) - self.assertIn( - email_subject, send_account_lockout_email.call_args[1]['args']) - self.assertEqual( - send_account_lockout_email.call_count, 2, "Called twice") + self.assertIn(email_subject, send_account_lockout_email.call_args[1]["args"]) + self.assertEqual(send_account_lockout_email.call_count, 2, "Called twice") # subsequent login fails after lockout even with correct credentials - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 401) - self.assertEqual(response.data['detail'], - u"Locked out. Too many wrong username/password " - u"attempts. Try again in 30 minutes.") + self.assertEqual( + response.data["detail"], + "Locked out. Too many wrong username/password " + "attempts. Try again in 30 minutes.", + ) # Other users on same IP not locked out - alice = User.objects.create(username='alice') - alice.set_password('alice') + alice = User.objects.create(username="alice") + alice.set_password("alice") update_partial_digests(alice, "alice") - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") request = self._get_request_session_with_auth(view, auth) response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(request.META.get('REMOTE_ADDR'), request_ip) + self.assertEqual(request.META.get("REMOTE_ADDR"), request_ip) # clear cache cache.clear() @@ -516,8 +525,8 @@ def test_generate_odk_token(self): """ Test that ODK Tokens can be created """ - view = ConnectViewSet.as_view({'post': 'odk_token'}) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) @@ -527,59 +536,51 @@ def test_regenerate_odk_token(self): Test that ODK Tokens can be regenerated and old tokens are set to Inactive after regeneration """ - view = ConnectViewSet.as_view({'post': 'odk_token'}) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - old_token = response.data['odk_token'] + old_token = response.data["odk_token"] with self.assertRaises(ODKToken.DoesNotExist): - ODKToken.objects.get( - user=self.user, status=ODKToken.INACTIVE) + ODKToken.objects.get(user=self.user, status=ODKToken.INACTIVE) - request = self.factory.post('/', **self.extra) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - self.assertNotEqual(response.data['odk_token'], old_token) + self.assertNotEqual(response.data["odk_token"], old_token) # Test that the previous token was set to inactive - inactive_token = ODKToken.objects.get( - user=self.user, status=ODKToken.INACTIVE) + inactive_token = ODKToken.objects.get(user=self.user, status=ODKToken.INACTIVE) self.assertEqual(inactive_token.raw_key, old_token) def test_retrieve_odk_token(self): """ Test that ODK Tokens can be retrieved """ - view = ConnectViewSet.as_view({ - 'post': 'odk_token', - 'get': 'odk_token' - }) - request = self.factory.post('/', **self.extra) + view = ConnectViewSet.as_view({"post": "odk_token", "get": "odk_token"}) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - odk_token = response.data['odk_token'] - expires = response.data['expires'] + odk_token = response.data["odk_token"] + expires = response.data["expires"] - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['odk_token'], odk_token) - self.assertEqual(response.data['expires'], expires) + self.assertEqual(response.data["odk_token"], odk_token) + self.assertEqual(response.data["expires"], expires) def test_deactivates_multiple_active_odk_token(self): """ Test that the viewset deactivates tokens when two or more are active at the same time and returns a new token """ - view = ConnectViewSet.as_view({ - 'post': 'odk_token', - 'get': 'odk_token' - }) + view = ConnectViewSet.as_view({"post": "odk_token", "get": "odk_token"}) # Create two active tokens token_1 = ODKToken.objects.create(user=self.user) @@ -590,18 +591,17 @@ def test_deactivates_multiple_active_odk_token(self): # Test that the GET request deactivates the two active tokens # and returns a new active token - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 200) - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) - self.assertNotEqual(response.data['odk_token'], token_1.raw_key) - self.assertNotEqual(response.data['odk_token'], token_2.raw_key) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) + self.assertNotEqual(response.data["odk_token"], token_1.raw_key) + self.assertNotEqual(response.data["odk_token"], token_2.raw_key) token_1 = ODKToken.objects.get(pk=token_1.pk) token_2 = ODKToken.objects.get(pk=token_2.pk) - token_3_key = response.data['odk_token'] + token_3_key = response.data["odk_token"] self.assertEqual(token_1.status, ODKToken.INACTIVE) self.assertEqual(token_2.status, ODKToken.INACTIVE) @@ -612,13 +612,11 @@ def test_deactivates_multiple_active_odk_token(self): token_1.status = ODKToken.ACTIVE token_1.save() - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 2) - request = self.factory.post('/', **self.extra) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 2) + request = self.factory.post("/", **self.extra) request.session = self.client.session response = view(request) self.assertEqual(response.status_code, 201) - self.assertEqual( - len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) - self.assertNotEqual(response.data['odk_token'], token_1.raw_key) - self.assertNotEqual(response.data['odk_token'], token_3_key) + self.assertEqual(len(ODKToken.objects.filter(status=ODKToken.ACTIVE)), 1) + self.assertNotEqual(response.data["odk_token"], token_1.raw_key) + self.assertNotEqual(response.data["odk_token"], token_3_key) diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index 97ab0f386c..fb50c68014 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -123,7 +123,7 @@ def authenticate(self, request): _( "Invalid username/password. " f"For security reasons, after {remaining_attempts} more failed " - "login attempts you'll have to wait {lockout_time} minutes " + f"login attempts you'll have to wait {lockout_time} minutes " "before trying again." ) ) From 0b7b1c7758588259ed4fce015ce99aa96e105da2 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 09:00:53 +0300 Subject: [PATCH 087/234] flake8 fixes --- onadata/apps/api/management/commands/fix_readonly_role_perms.py | 1 - onadata/libs/renderers/renderers.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 1114bc4754..4e2eab79c4 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -128,7 +128,6 @@ def handle(self, *args, **options): if len(args) < 3: raise CommandError("Param not set. ") - app = args[0] model = args[1] username = args[2] new_perms = list(args[3:]) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 3c4833736d..219abaa8f3 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -121,6 +121,7 @@ class XLSRenderer(BaseRenderer): media_type = "application/vnd.openxmlformats" format = "xls" charset = None + # pylint: disable=no-self-use,unused-argument def render(self, data, accepted_media_type=None, renderer_context=None): """ From bd9c41a29a1c405cf296fcf792462003ca730c24 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 10:10:02 +0300 Subject: [PATCH 088/234] batch: cleanup --- .../tests/viewsets/test_abstract_viewset.py | 188 ++-- .../viewsets/test_xform_submission_viewset.py | 968 ++++++++++-------- onadata/apps/api/viewsets/data_viewset.py | 11 +- 3 files changed, 673 insertions(+), 494 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 9bee9ab54b..f3a3180e02 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -1,17 +1,20 @@ +# -*- coding: utf-8 -*- +""" +Test base class for API viewset tests. +""" import json import os import re from tempfile import NamedTemporaryFile -from builtins import open from django.conf import settings -from django.contrib.auth import authenticate -from django.contrib.auth.models import Permission, User +from django.contrib.auth import authenticate, get_user_model +from django.contrib.auth.models import Permission from django.test import TestCase + from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth from httmock import HTTMock -from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from rest_framework.test import APIRequestFactory from onadata.apps.api.models import OrganizationProfile, Team @@ -32,11 +35,47 @@ from onadata.apps.main.models import MetaData, UserProfile from onadata.apps.viewer.models import DataDictionary from onadata.libs.serializers.project_serializer import ProjectSerializer +from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from onadata.libs.utils.common_tools import merge_dicts from onadata.libs.utils.user_auth import get_user_default_project +# pylint: disable=invalid-name +User = get_user_model() + + +def _set_api_permissions(user): + add_userprofile = Permission.objects.get( + content_type__app_label="main", + content_type__model="userprofile", + codename="add_userprofile", + ) + user.user_permissions.add(add_userprofile) + + +def add_uuid_to_submission_xml(path, xform): + """ + Adds the formhub uuid to an XML XForm submission at the given path. + """ + with NamedTemporaryFile(delete=False, mode="w") as tmp_file: + split_xml = None + + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) + + split_xml[1:1] = [f"{xform.uuid}"] + tmp_file.write("".join(split_xml)) + path = tmp_file.name + + return path + + +# pylint: disable=too-many-instance-attributes class TestAbstractViewSet(PyxformMarkdown, TestCase): + """ + Base test class for API viewsets. + """ + surveys = [ "transport_2011-07-25_19-05-49", "transport_2011-07-25_19-05-36", @@ -88,17 +127,10 @@ def user_profile_data(self): "name": "Bob erama", } - def _set_api_permissions(self, user): - add_userprofile = Permission.objects.get( - content_type__app_label="main", - content_type__model="userprofile", - codename="add_userprofile", - ) - user.user_permissions.add(add_userprofile) - - def _create_user_profile(self, extra_post_data={}): + def _create_user_profile(self, extra_post_data=None): + extra_post_data = {} if extra_post_data is None else extra_post_data self.profile_data = merge_dicts(self.profile_data, extra_post_data) - user, created = User.objects.get_or_create( + user, _created = User.objects.get_or_create( username=self.profile_data["username"], first_name=self.profile_data["first_name"], last_name=self.profile_data["last_name"], @@ -106,7 +138,7 @@ def _create_user_profile(self, extra_post_data={}): ) user.set_password(self.profile_data["password1"]) user.save() - new_profile, created = UserProfile.objects.get_or_create( + new_profile, _created = UserProfile.objects.get_or_create( user=user, name=self.profile_data["first_name"], city=self.profile_data["city"], @@ -119,7 +151,8 @@ def _create_user_profile(self, extra_post_data={}): return new_profile - def _login_user_and_profile(self, extra_post_data={}): + def _login_user_and_profile(self, extra_post_data=None): + extra_post_data = {} if extra_post_data is None else extra_post_data profile = self._create_user_profile(extra_post_data) self.user = profile.user self.assertTrue( @@ -127,9 +160,10 @@ def _login_user_and_profile(self, extra_post_data={}): username=self.user.username, password=self.profile_data["password1"] ) ) - self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} - def _org_create(self, org_data={}): + def _org_create(self, org_data=None): + org_data = {} if org_data is None else org_data view = OrganizationProfileViewSet.as_view({"get": "list", "post": "create"}) request = self.factory.get("/", **self.extra) response = view(request) @@ -155,10 +189,11 @@ def _org_create(self, org_data={}): ) response = view(request) self.assertEqual(response.status_code, 201) - data["url"] = "http://testserver/api/v1/orgs/%s" % data["org"] - data["user"] = "http://testserver/api/v1/users/%s" % data["org"] + data["url"] = "http://testserver/api/v1/orgs/{data['org']}" + data["user"] = "http://testserver/api/v1/users/{data['org']}" data["creator"] = "http://testserver/api/v1/users/bob" self.assertDictContainsSubset(data, response.data) + # pylint: disable=attribute-defined-outside-init self.company_data = response.data self.organization = OrganizationProfile.objects.get(user__username=data["org"]) @@ -183,17 +218,18 @@ def _publish_form_with_hxl_support(self): "fixtures", "hxl_example", "instances", - "instance_%s.xml" % x, + f"instance_{x}.xml", ) self._make_submission(path) - def _project_create(self, project_data={}, merge=True): + def _project_create(self, project_data=None, merge=True): + project_data = {} if project_data is None else project_data view = ProjectViewSet.as_view({"post": "create"}) if merge: data = { "name": "demo", - "owner": "http://testserver/api/v1/users/%s" % self.user.username, + "owner": f"http://testserver/api/v1/users/{self.user.username}", "metadata": { "description": "Some description", "location": "Naivasha, Kenya", @@ -210,6 +246,7 @@ def _project_create(self, project_data={}, merge=True): ) response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.project = Project.objects.filter(name=data["name"], created_by=self.user)[ 0 ] @@ -217,13 +254,15 @@ def _project_create(self, project_data={}, merge=True): self.assertDictContainsSubset(data, response.data) request.user = self.user + # pylint: disable=attribute-defined-outside-init self.project_data = ProjectSerializer( self.project, context={"request": request} ).data def _publish_xls_form_to_project( - self, publish_data={}, merge=True, public=False, xlsform_path=None + self, publish_data=None, merge=True, public=False, xlsform_path=None ): + publish_data = {} if publish_data is None else publish_data if not hasattr(self, "project"): self._project_create() elif self.project.created_by != self.user: @@ -234,8 +273,10 @@ def _publish_xls_form_to_project( project_id = self.project.pk if merge: data = { - "owner": "http://testserver/api/v1/users/%s" - % self.project.organization.username, + "owner": ( + "http://testserver/api/v1/users/" + f"{self.project.organization.username}" + ), "public": False, "public_data": False, "description": "transportation_2011_07_25", @@ -267,30 +308,17 @@ def _publish_xls_form_to_project( request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.xform = XForm.objects.all().order_by("pk").reverse()[0] - data.update( - {"url": "http://testserver/api/v1/forms/%s" % (self.xform.pk)} - ) + data.update({"url": f"http://testserver/api/v1/forms/{self.xform.pk}"}) # Input was a private so change to public if project public if public: data["public_data"] = data["public"] = True + # pylint: disable=attribute-defined-outside-init self.form_data = response.data - def _add_uuid_to_submission_xml(self, path, xform): - tmp_file = NamedTemporaryFile(delete=False, mode="w") - split_xml = None - - with open(path, encoding="utf-8") as _file: - split_xml = re.split(r"()", _file.read()) - - split_xml[1:1] = ["%s" % xform.uuid] - tmp_file.write("".join(split_xml)) - path = tmp_file.name - tmp_file.close() - - return path - + # pylint: disable=too-many-arguments,too-many-locals,unused-argument def _make_submission( self, path, @@ -311,25 +339,26 @@ def _make_submission( tmp_file = None if add_uuid: - path = self._add_uuid_to_submission_xml(path, self.xform) + path = add_uuid_to_submission_xml(path, self.xform) with open(path, encoding="utf-8") as f: post_data = {"xml_submission_file": f} if media_file is not None: if isinstance(media_file, list): - for c in range(len(media_file)): - post_data["media_file_{}".format(c)] = media_file[c] + for position, _value in enumerate(media_file): + post_data[f"media_file_{position}"] = media_file[position] else: post_data["media_file"] = media_file if username is None: username = self.user.username - url_prefix = "%s/" % username if username else "" - url = "/%ssubmission" % url_prefix + url_prefix = f"{username if username else ''}/" + url = f"/{url_prefix}submission" request = self.factory.post(url, post_data) request.user = authenticate(username=auth.username, password=auth.password) + # pylint: disable=attribute-defined-outside-init self.response = submission(request, username=username) if auth and self.response.status_code == 401: @@ -409,6 +438,7 @@ def _submit_transport_instance_w_attachment( ) attachment = Attachment.objects.all().reverse()[0] + # pylint: disable=attribute-defined-outside-init self.attachment = attachment def _post_metadata(self, data, test=True): @@ -422,6 +452,7 @@ def _post_metadata(self, data, test=True): self.assertEqual(response.status_code, 201, response.data) another_count = MetaData.objects.count() self.assertEqual(another_count, count + 1) + # pylint: disable=attribute-defined-outside-init self.metadata = MetaData.objects.get(pk=response.data["id"]) self.metadata_data = response.data @@ -459,8 +490,8 @@ def _create_dataview(self, data=None, project=None, xform=None): if not data: data = { "name": "My DataView", - "xform": "http://testserver/api/v1/forms/%s" % xform.pk, - "project": "http://testserver/api/v1/projects/%s" % project.pk, + "xform": f"http://testserver/api/v1/forms/{xform.pk}", + "project": f"http://testserver/api/v1/projects/{project.pk}", "columns": '["name", "age", "gender"]', "query": '[{"column":"age","filter":">","value":"20"},' '{"column":"age","filter":"<","value":"50"}]', @@ -468,19 +499,20 @@ def _create_dataview(self, data=None, project=None, xform=None): request = self.factory.post("/", data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # load the created dataview + # pylint: disable=attribute-defined-outside-init self.data_view = DataView.objects.filter(xform=xform, project=project).last() - self.assertEquals(response.data["name"], data["name"]) - self.assertEquals(response.data["xform"], data["xform"]) - self.assertEquals(response.data["project"], data["project"]) - self.assertEquals(response.data["columns"], json.loads(data["columns"])) - self.assertEquals( + self.assertEqual(response.data["name"], data["name"]) + self.assertEqual(response.data["xform"], data["xform"]) + self.assertEqual(response.data["project"], data["project"]) + self.assertEqual(response.data["columns"], json.loads(data["columns"])) + self.assertEqual( response.data["query"], json.loads(data["query"]) if "query" in data else {} ) - self.assertEquals( + self.assertEqual( response.data["url"], f"http://testserver/api/v1/dataviews/{self.data_view.pk}", ) @@ -507,23 +539,24 @@ def _create_widget(self, data=None, group_by=""): ) response = view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, Widget.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, Widget.objects.all().count()) + # pylint: disable=attribute-defined-outside-init self.widget = Widget.objects.all().order_by("pk").reverse()[0] - self.assertEquals(response.data["id"], self.widget.id) - self.assertEquals(response.data["title"], data.get("title")) - self.assertEquals(response.data["content_object"], data["content_object"]) - self.assertEquals(response.data["widget_type"], data["widget_type"]) - self.assertEquals(response.data["view_type"], data["view_type"]) - self.assertEquals(response.data["column"], data["column"]) - self.assertEquals(response.data["description"], data.get("description")) - self.assertEquals(response.data["group_by"], data.get("group_by")) - self.assertEquals(response.data["aggregation"], data.get("aggregation")) - self.assertEquals(response.data["order"], self.widget.order) - self.assertEquals(response.data["data"], []) - self.assertEquals(response.data["metadata"], data.get("metadata", {})) + self.assertEqual(response.data["id"], self.widget.id) + self.assertEqual(response.data["title"], data.get("title")) + self.assertEqual(response.data["content_object"], data["content_object"]) + self.assertEqual(response.data["widget_type"], data["widget_type"]) + self.assertEqual(response.data["view_type"], data["view_type"]) + self.assertEqual(response.data["column"], data["column"]) + self.assertEqual(response.data["description"], data.get("description")) + self.assertEqual(response.data["group_by"], data.get("group_by")) + self.assertEqual(response.data["aggregation"], data.get("aggregation")) + self.assertEqual(response.data["order"], self.widget.order) + self.assertEqual(response.data["data"], []) + self.assertEqual(response.data["metadata"], data.get("metadata", {})) def _team_create(self): self._org_create() @@ -536,21 +569,25 @@ def _team_create(self): ) response = view(request) self.assertEqual(response.status_code, 201) + # pylint: disable=attribute-defined-outside-init self.owner_team = Team.objects.get( organization=self.organization.user, - name="%s#Owners" % (self.organization.user.username), + name=f"{self.organization.user.username}#Owners", ) team = Team.objects.get( organization=self.organization.user, - name="%s#%s" % (self.organization.user.username, data["name"]), + name=f"{self.organization.user.username}#{data['name']}", ) - data["url"] = "http://testserver/api/v1/teams/%s" % team.pk + data["url"] = f"http://testserver/api/v1/teams/{team.pk}" data["teamid"] = team.id self.assertDictContainsSubset(data, response.data) self.team_data = response.data self.team = team def is_sorted_desc(self, s): + """ + Returns True if a list is sorted in descending order. + """ if len(s) in [0, 1]: return True if s[0] >= s[1]: @@ -558,6 +595,9 @@ def is_sorted_desc(self, s): return False def is_sorted_asc(self, s): + """ + Returns True if a list is sorted in ascending order. + """ if len(s) in [0, 1]: return True if s[0] <= s[1]: diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py index c6dcb9d915..66b962f63a 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py @@ -15,10 +15,11 @@ import simplejson as json from django_digest.test import DigestAuth -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.viewsets.xform_submission_viewset import \ - XFormSubmissionViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + add_uuid_to_submission_xml, +) +from onadata.apps.api.viewsets.xform_submission_viewset import XFormSubmissionViewSet from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.libs.permissions import DataEntryRole from onadata.libs.utils.common_tools import get_uuid @@ -29,12 +30,10 @@ class TestXFormSubmissionViewSet(TestAbstractViewSet, TransactionTestCase): """ TestXFormSubmissionViewSet test class. """ + def setUp(self): super(TestXFormSubmissionViewSet, self).setUp() - self.view = XFormSubmissionViewSet.as_view({ - "head": "create", - "post": "create" - }) + self.view = XFormSubmissionViewSet.as_view({"head": "create", "post": "create"}) self._publish_xls_form_to_project() def test_unique_instanceid_per_form_only(self): @@ -42,11 +41,10 @@ def test_unique_instanceid_per_form_only(self): Test unique instanceID submissions per form. """ self._make_submissions() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice = self._create_user_profile(alice_data) self.user = alice.user - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} self._publish_xls_form_to_project() self._make_submissions() @@ -56,31 +54,40 @@ def test_post_submission_anonymous(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - '/%s/submission' % self.user.username, data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/%s/submission" % self.user.username, data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/%s/submission' - % self.user.username) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual( + response["Location"], + "http://testserver/%s/submission" % self.user.username, + ) def test_username_case_insensitive(self): """ @@ -88,39 +95,47 @@ def test_username_case_insensitive(self): to the username case """ # Change username to Bob - self.user.username = 'Bob' + self.user.username = "Bob" self.user.save() survey = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', survey, media_file) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + media_file, + ) form = XForm.objects.get(user=self.user) count = form.submission_count() - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', survey, survey + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} + self.main_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} # Make submission to /bob/submission - request = self.factory.post( - '/%s/submission' % 'bob', data) + request = self.factory.post("/%s/submission" % "bob", data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) # Submission should be submitted to the right form - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(count+1, form.submission_count()) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(count + 1, form.submission_count()) def test_post_submission_authenticated(self): """ @@ -128,58 +143,78 @@ def test_post_submission_authenticated(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_uuid_other_user_username_not_provided(self): """ Test submission without formhub/uuid done by a different user who has no permission to the form fails. """ - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - path = self._add_uuid_to_submission_xml(path, self.xform) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + path = add_uuid_to_submission_xml(path, self.xform) - with open(path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + with open(path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) self.assertEqual(response.status_code, 404) @@ -190,23 +225,34 @@ def test_post_submission_uuid_duplicate_no_username_provided(self): from the original is properly routed to the request users version of the form """ - alice_profile = self._create_user_profile({ - 'username': 'alice', - 'email': 'alice@localhost.com' - }) + alice_profile = self._create_user_profile( + {"username": "alice", "email": "alice@localhost.com"} + ) s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - path = self._add_uuid_to_submission_xml(path, self.xform) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + path = add_uuid_to_submission_xml(path, self.xform) - with open(path, 'rb') as sf: + with open(path, "rb") as sf: # Submits to the correct form count = XForm.objects.count() original_form_pk = self.xform.pk @@ -221,21 +267,19 @@ def test_post_submission_uuid_duplicate_no_username_provided(self): self.assertEqual(XForm.objects.count(), count + 1) request = self.factory.post( - '/submission', - {'xml_submission_file': sf, 'media_file': f}) + "/submission", {"xml_submission_file": sf, "media_file": f} + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) self.assertEqual(response.status_code, 201) duplicate_form.refresh_from_db() + self.assertEqual(duplicate_form.instances.all().count(), 1) self.assertEqual( - duplicate_form.instances.all().count(), - 1) - self.assertEqual( - XForm.objects.get( - pk=original_form_pk).instances.all().count(), 0) + XForm.objects.get(pk=original_form_pk).instances.all().count(), 0 + ) def test_post_submission_authenticated_json(self): """ @@ -243,28 +287,25 @@ def test_post_submission_authenticated_json(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json_list(self): """ @@ -273,28 +314,25 @@ def test_post_submission_authenticated_bad_json_list(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', [data], format='json') + request = self.factory.post("/submission", [data], format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Invalid data', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Invalid data", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json_submission_list(self): """ @@ -303,29 +341,26 @@ def test_post_submission_authenticated_bad_json_submission_list(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - data['submission'] = [data['submission']] - request = self.factory.post('/submission', data, format='json') + data["submission"] = [data["submission"]] + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Incorrect format', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Incorrect format", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_authenticated_bad_json(self): """ @@ -333,29 +368,26 @@ def test_post_submission_authenticated_bad_json(self): """ path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - '..', - 'fixtures', - 'transport_submission_bad.json') - with open(path, encoding='utf-8') as f: + "..", + "fixtures", + "transport_submission_bad.json", + ) + with open(path, encoding="utf-8") as f: data = json.loads(f.read()) - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - request = self.factory.post('/submission', data, format='json') - auth = DigestAuth('bob', 'bobbob') + request = self.factory.post("/submission", data, format="json") + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'Received empty submission', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'application/json') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Received empty submission", status_code=400) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "application/json") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_require_auth(self): """ @@ -365,28 +397,29 @@ def test_post_submission_require_auth(self): self.user.profile.save() submission = self.surveys[0] submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', submission, submission + '.xml') - with open(submission_path, 'rb') as submission_file: - data = {'xml_submission_file': submission_file} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + submission, + submission + ".xml", + ) + with open(submission_path, "rb") as submission_file: + data = {"xml_submission_file": submission_file} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') - self.assertEqual(response['Location'], - 'http://testserver/submission') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_submission_require_auth_anonymous_user(self): """ @@ -398,17 +431,29 @@ def test_post_submission_require_auth_anonymous_user(self): count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) @@ -423,35 +468,48 @@ def test_post_submission_require_auth_other_user(self): self.user.profile.require_auth = True self.user.profile.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) self.assertEqual(count, Attachment.objects.count()) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) self.assertContains( response, - 'alice is not allowed to make submissions to bob', - status_code=403) + "alice is not allowed to make submissions to bob", + status_code=403, + ) def test_post_submission_require_auth_data_entry_role(self): """ @@ -461,63 +519,81 @@ def test_post_submission_require_auth_data_entry_role(self): self.user.profile.require_auth = True self.user.profile.save() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) DataEntryRole.add(alice_profile.user, self.xform) count = Attachment.objects.count() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post('/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 401) self.assertEqual(count, Attachment.objects.count()) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) - self.assertContains(response, 'Successful submission', - status_code=201) + self.assertContains(response, "Successful submission", status_code=201) def test_post_submission_json_without_submission_key(self): """ Tesut JSON submission without the submission key fails. """ data = {"id": "transportation_2011_07_25"} - request = self.factory.post('/submission', data, format='json') + request = self.factory.post("/submission", data, format="json") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request) - self.assertContains(response, 'No submission key provided.', - status_code=400) + self.assertContains(response, "No submission key provided.", status_code=400) def test_NaN_in_submission(self): """ Test submissions with uuid as NaN are successful. """ xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid_NaN', 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid_NaN", + "submission.xml", + ) self._make_submission(path) def test_rapidpro_post_submission(self): @@ -525,112 +601,112 @@ def test_rapidpro_post_submission(self): Test a Rapidpro Webhook POST submission. """ # pylint: disable=C0301 - data = json.dumps({ - 'contact': { - 'name': 'Davis', - 'urn': 'tel:+12065551212', - 'uuid': '23dae14f-7202-4ff5-bdb6-2390d2769968' - }, - 'flow': { - 'name': '1166', - 'uuid': '9da5e439-35af-4ecb-b7fc-2911659f6b04' - }, - 'results': { - 'fruit_name': { - 'category': 'All Responses', - 'value': 'orange' + data = json.dumps( + { + "contact": { + "name": "Davis", + "urn": "tel:+12065551212", + "uuid": "23dae14f-7202-4ff5-bdb6-2390d2769968", + }, + "flow": { + "name": "1166", + "uuid": "9da5e439-35af-4ecb-b7fc-2911659f6b04", + }, + "results": { + "fruit_name": {"category": "All Responses", "value": "orange"}, }, } - }) + ) request = self.factory.post( - '/submission', data, - content_type='application/json', - HTTP_USER_AGENT='RapidProMailroom/5.2.0') + "/submission", + data, + content_type="application/json", + HTTP_USER_AGENT="RapidProMailroom/5.2.0", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") # InstanceID is returned as uuid:. # Retrieving the uuid without the prefix in order to retrieve # the Instance object - uuid = response.data.get('instanceID').split(':')[1] + uuid = response.data.get("instanceID").split(":")[1] instance = Instance.objects.get(uuid=uuid) expected_xml = ( f"<{self.xform.survey.name} id=" "'transportation_2011_07_25'>orange" f"\n\n uuid:{uuid}" - f"\n") + f"\n" + ) self.assertEqual(instance.xml, expected_xml) - self.assertEqual(self.xform.survey.name, 'data') + self.assertEqual(self.xform.survey.name, "data") def test_legacy_rapidpro_post_submission(self): """ Test a Legacy Rapidpro Webhook POST submission. """ # pylint: disable=C0301 - data = 'run=76250&text=orange&flow_uuid=9da5e439-35af-4ecb-b7fc-2911659f6b04&phone=%2B12065550100&step=3b15df81-a0bd-4de7-8186-145ea3bb6c43&contact_name=Antonate+Maritim&flow_name=fruit&header=Authorization&urn=tel%3A%2B12065550100&flow=1166&relayer=-1&contact=fe4df540-39c1-4647-b4bc-1c93833e22e0&values=%5B%7B%22category%22%3A+%7B%22base%22%3A+%22All+Responses%22%7D%2C+%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22time%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22rule_value%22%3A+%22orange%22%2C+%22value%22%3A+%22orange%22%2C+%22label%22%3A+%22fruit_name%22%7D%5D&time=2017-10-04T07%3A18%3A08.205524Z&steps=%5B%7B%22node%22%3A+%220e18202f-9ec4-4756-b15b-e9f152122250%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.548657Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22text%22%3A+%22Fruit%3F%22%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%2C+%7B%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22type%22%3A+%22R%22%2C+%22value%22%3A+%22orange%22%7D%2C+%7B%22node%22%3A+%223b15df81-a0bd-4de7-8186-145ea3bb6c43%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22left_on%22%3A+null%2C+%22text%22%3A+null%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%5D&flow_base_language=base&channel=-1' # noqa + data = "run=76250&text=orange&flow_uuid=9da5e439-35af-4ecb-b7fc-2911659f6b04&phone=%2B12065550100&step=3b15df81-a0bd-4de7-8186-145ea3bb6c43&contact_name=Antonate+Maritim&flow_name=fruit&header=Authorization&urn=tel%3A%2B12065550100&flow=1166&relayer=-1&contact=fe4df540-39c1-4647-b4bc-1c93833e22e0&values=%5B%7B%22category%22%3A+%7B%22base%22%3A+%22All+Responses%22%7D%2C+%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22time%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22rule_value%22%3A+%22orange%22%2C+%22value%22%3A+%22orange%22%2C+%22label%22%3A+%22fruit_name%22%7D%5D&time=2017-10-04T07%3A18%3A08.205524Z&steps=%5B%7B%22node%22%3A+%220e18202f-9ec4-4756-b15b-e9f152122250%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.548657Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22text%22%3A+%22Fruit%3F%22%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%2C+%7B%22node%22%3A+%228037c12f-a277-4255-b630-6a03b035767a%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A15%3A17.604668Z%22%2C+%22left_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22text%22%3A+%22orange%22%2C+%22type%22%3A+%22R%22%2C+%22value%22%3A+%22orange%22%7D%2C+%7B%22node%22%3A+%223b15df81-a0bd-4de7-8186-145ea3bb6c43%22%2C+%22arrived_on%22%3A+%222017-10-04T07%3A18%3A08.171069Z%22%2C+%22left_on%22%3A+null%2C+%22text%22%3A+null%2C+%22type%22%3A+%22A%22%2C+%22value%22%3A+null%7D%5D&flow_base_language=base&channel=-1" # noqa request = self.factory.post( - '/submission', data, - content_type='application/x-www-form-urlencoded') + "/submission", data, content_type="application/x-www-form-urlencoded" + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_post_empty_submission(self): """ Test empty submission fails. """ - request = self.factory.post( - '/%s/submission' % self.user.username, {}) + request = self.factory.post("/%s/submission" % self.user.username, {}) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'No XML submission file.', - status_code=400) + self.assertContains(response, "No XML submission file.", status_code=400) def test_auth_submission_head_request(self): """ Test HEAD submission request. """ - request = self.factory.head('/submission') + request = self.factory.head("/submission") response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 204, response.data) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_head_submission_anonymous(self): """ Test HEAD submission request for anonymous user. """ - request = self.factory.head('/%s/submission' % self.user.username) + request = self.factory.head("/%s/submission" % self.user.username) request.user = AnonymousUser() response = self.view(request, username=self.user.username) self.assertEqual(response.status_code, 204, response.data) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], - 'http://testserver/%s/submission' - % self.user.username) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual( + response["Location"], "http://testserver/%s/submission" % self.user.username + ) def test_floip_format_submission(self): """ @@ -639,17 +715,20 @@ def test_floip_format_submission(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_floip_format_submission_missing_column(self): """ @@ -658,16 +737,22 @@ def test_floip_format_submission_missing_column(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, "Wrong number of values (5) in row 0, " - "expecting 6 values", status_code=400) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains( + response, + "Wrong number of values (5) in row 0, " "expecting 6 values", + status_code=400, + ) def test_floip_format_submission_not_list(self): """ @@ -676,17 +761,21 @@ def test_floip_format_submission_not_list(self): # pylint: disable=C0301 data = '{"option_order": ["male", "female"]}' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {u'non_field_errors': [ - u'Invalid format. Expecting a list.']}) + self.assertEqual( + response.data, {"non_field_errors": ["Invalid format. Expecting a list."]} + ) def test_floip_format_submission_is_valid_json(self): """ @@ -695,14 +784,17 @@ def test_floip_format_submission_is_valid_json(self): # pylint: disable=C0301 data = '"2017-05-23T13:35:37.119-04:00", 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) self.assertContains(response, "Extra data", status_code=400) def test_floip_format_multiple_rows_submission(self): @@ -712,17 +804,20 @@ def test_floip_format_multiple_rows_submission(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}], ["2017-05-23T13:35:47.822-04:00", 20394823950, 923842093, "ae54d7", "chocolate", null ]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', status_code=201) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], 'http://testserver/submission') + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Location"], "http://testserver/submission") def test_floip_format_multiple_rows_instance(self): """ @@ -731,20 +826,24 @@ def test_floip_format_multiple_rows_instance(self): # pylint: disable=C0301 data = '[["2017-05-23T13:35:37.119-04:00", 20394823948, 923842093, "ae54d3", "female", {"option_order": ["male", "female"]}], ["2017-05-23T13:35:47.822-04:00", 20394823950, 923842093, "ae54d7", "chocolate", null ]]' # noqa request = self.factory.post( - '/submission', data, - content_type='application/vnd.org.flowinterop.results+json') + "/submission", + data, + content_type="application/vnd.org.flowinterop.results+json", + ) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) - response = self.view(request, username=self.user.username, - xform_pk=self.xform.pk) + response = self.view( + request, username=self.user.username, xform_pk=self.xform.pk + ) instance_json = Instance.objects.last().json data_responses = [i[4] for i in json.loads(data)] - self.assertTrue(any(i in data_responses - for i in instance_json.values())) + self.assertTrue(any(i in data_responses for i in instance_json.values())) - @mock.patch('onadata.apps.api.viewsets.xform_submission_viewset.SubmissionSerializer') # noqa + @mock.patch( + "onadata.apps.api.viewsets.xform_submission_viewset.SubmissionSerializer" + ) # noqa def test_post_submission_unreadable_post_error(self, MockSerializer): """ Test UnreadablePostError exception during submission.. @@ -752,23 +851,35 @@ def test_post_submission_unreadable_post_error(self, MockSerializer): MockSerializer.side_effect = UnreadablePostError() s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - '/%s/submission' % self.user.username, data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post("/%s/submission" % self.user.username, data) request.user = AnonymousUser() response = self.view(request, username=self.user.username) - self.assertContains(response, 'Unable to read submitted file', - status_code=400) - self.assertTrue(response.has_header('X-OpenRosa-Version')) + self.assertContains( + response, "Unable to read submitted file", status_code=400 + ) + self.assertTrue(response.has_header("X-OpenRosa-Version")) def test_post_submission_using_pk_while_anonymous(self): """ @@ -778,35 +889,44 @@ def test_post_submission_using_pk_while_anonymous(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} - request = self.factory.post( - f'/enketo/{self.xform.pk}/submission', data) + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} + request = self.factory.post(f"/enketo/{self.xform.pk}/submission", data) count = Instance.objects.filter(xform=self.xform).count() request.user = AnonymousUser() response = self.view(request, xform_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") self.assertEqual( - response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) self.assertEqual( - Instance.objects.filter(xform=self.xform).count(), - count+1) + Instance.objects.filter(xform=self.xform).count(), count + 1 + ) def test_head_submission_request_w_no_auth(self): """ @@ -819,37 +939,48 @@ def test_head_submission_request_w_no_auth(self): s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb'): + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb"): # When require_auth is enabled and # no auth is passed to the request should fail - request = self.factory.head( - f'/enketo/{self.xform.pk}/submission') + request = self.factory.head(f"/enketo/{self.xform.pk}/submission") response = self.view(request) self.assertEqual(response.status_code, 401) # When require_auth is enabled & no auth passed is ok - request = self.factory.head( - f'/enketo/{self.xform.pk}/submission') + request = self.factory.head(f"/enketo/{self.xform.pk}/submission") request.user = self.xform.user response = self.view(request, xform_pk=self.xform.pk) self.assertEqual(response.status_code, 401) - self.assertTrue(response.has_header('X-OpenRosa-Version')) + self.assertTrue(response.has_header("X-OpenRosa-Version")) # Test Content-Length header is available - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') # noqa + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual( + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) # noqa def test_post_submission_using_pk_while_authenticated(self): """ @@ -859,35 +990,44 @@ def test_post_submission_using_pk_while_authenticated(self): """ s = self.surveys[0] media_file = "1335783522563.jpg" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg', - os.path.getsize(path), None) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + f = InMemoryUploadedFile( + f, "media_file", media_file, "image/jpg", os.path.getsize(path), None + ) submission_path = os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml') - with open(submission_path, 'rb') as sf: - data = {'xml_submission_file': sf, 'media_file': f} + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + with open(submission_path, "rb") as sf: + data = {"xml_submission_file": sf, "media_file": f} count = Instance.objects.filter(xform=self.xform).count() - request = self.factory.post( - f'/enketo/{self.xform.pk}/submission', data) + request = self.factory.post(f"/enketo/{self.xform.pk}/submission", data) response = self.view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = self.view(request, form_pk=self.xform.pk) - self.assertContains(response, 'Successful submission', - status_code=201) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) - self.assertEqual(response['Content-Type'], - 'text/xml; charset=utf-8') + self.assertContains(response, "Successful submission", status_code=201) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) + self.assertEqual(response["Content-Type"], "text/xml; charset=utf-8") self.assertEqual( - response['Location'], - f'http://testserver/enketo/{self.xform.pk}/submission') + response["Location"], + f"http://testserver/enketo/{self.xform.pk}/submission", + ) self.assertEqual( - Instance.objects.filter(xform=self.xform).count(), - count + 1) + Instance.objects.filter(xform=self.xform).count(), count + 1 + ) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 74ba698d9a..b6d2bf2bc4 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -5,7 +5,6 @@ import json import math import types -from builtins import str as text from typing import Union from django.conf import settings @@ -81,7 +80,7 @@ def get_data_and_form(kwargs): """ Checks if the dataid in ``kwargs`` is a valid integer. """ - data_id = text(kwargs.get("dataid")) + data_id = str(kwargs.get("dataid")) if not data_id.isdigit(): raise ParseError(_("Data ID should be an integer")) @@ -99,7 +98,7 @@ def delete_instance(instance, user): try: instance.set_deleted(timezone.now(), user) except FormInactiveError as e: - raise ParseError(text(e)) from e + raise ParseError(str(e)) from e # pylint: disable=too-many-ancestors @@ -324,7 +323,7 @@ def enketo(self, request, *args, **kwargs): if "edit_url" in data: data["url"] = data.pop("edit_url") except EnketoError as e: - raise ParseError(text(e)) from e + raise ParseError(str(e)) from e else: raise PermissionDenied(_("You do not have edit permissions.")) @@ -676,9 +675,9 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): (get_etag_hash_from_query(records, sql, params)), ) except ValueError as e: - raise ParseError(text(e)) from e + raise ParseError(str(e)) from e except DataError as e: - raise ParseError(text(e)) from e + raise ParseError(str(e)) from e def paginate_queryset(self, queryset): if self.paginator is None: From 2eb4e70bda194e106d856f257412df192be4cb14 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 10:10:49 +0300 Subject: [PATCH 089/234] Fix: handle DataError exceptions --- onadata/apps/logger/models/data_view.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 279c0898ed..3f187f345d 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -8,10 +8,10 @@ from django.conf import settings from django.contrib.gis.db import models from django.db import connection -from django.db.models import JSONField from django.db.models.signals import post_delete, post_save from django.utils import timezone from django.utils.translation import ugettext as _ +from django.db.utils import DataError from onadata.apps.viewer.parsed_instance_tools import get_where_clause from onadata.libs.models.sorting import ( # noqa pylint: disable=unused-import @@ -105,8 +105,8 @@ class DataView(models.Model): xform = models.ForeignKey("logger.XForm", on_delete=models.CASCADE) project = models.ForeignKey("logger.Project", on_delete=models.CASCADE) - columns = JSONField() - query = JSONField(default=dict, blank=True) + columns = models.JSONField() + query = models.JSONField(default=dict, blank=True) instances_with_geopoints = models.BooleanField(default=False) matches_parent = models.BooleanField(default=False) date_created = models.DateTimeField(auto_now_add=True) @@ -415,7 +415,10 @@ def query_data( # pylint: disable=too-many-arguments filter_query, ) - records = list(DataView.query_iterator(sql, columns, params, count)) + try: + records = list(DataView.query_iterator(sql, columns, params, count)) + except DataError as e: + return {"error": _(str(e))} return records From 0a4e07591cd2010858314065587e13e677d78675 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 10:16:09 +0300 Subject: [PATCH 090/234] assertEquals is deprecated --- onadata/apps/api/tests/models/test_project.py | 10 +- .../apps/api/tests/models/test_temp_token.py | 4 +- .../api/tests/viewsets/test_data_viewset.py | 58 ++++---- .../viewsets/test_merged_xform_viewset.py | 6 +- .../tests/viewsets/test_metadata_viewset.py | 10 +- .../apps/api/tests/viewsets/test_ona_api.py | 4 +- .../test_organization_profile_viewset.py | 14 +- .../api/tests/viewsets/test_osm_viewset.py | 2 +- .../api/tests/viewsets/test_stats_viewset.py | 2 +- .../api/tests/viewsets/test_team_viewset.py | 2 +- .../api/tests/viewsets/test_user_viewset.py | 16 +-- .../api/tests/viewsets/test_widget_viewset.py | 122 ++++++++-------- .../api/tests/viewsets/test_xform_viewset.py | 130 +++++++++--------- .../logger/tests/models/test_data_view.py | 16 +-- onadata/apps/logger/tests/test_form_list.py | 2 +- .../apps/logger/tests/test_form_submission.py | 18 +-- onadata/apps/logger/tests/test_parsing.py | 2 +- .../logger/tests/test_simple_submission.py | 14 +- onadata/apps/main/tests/test_csv_export.py | 2 +- onadata/apps/main/tests/test_form_auth.py | 2 +- onadata/apps/main/tests/test_form_errors.py | 6 +- onadata/apps/main/tests/test_form_metadata.py | 10 +- onadata/apps/main/tests/test_metadata.py | 16 +-- onadata/apps/main/tests/test_process.py | 6 +- .../test_user_id_string_unique_together.py | 6 +- .../messaging/tests/test_backends_mqtt.py | 12 +- .../restservice/tests/test_restservice.py | 10 +- .../viewsets/test_restservicesviewset.py | 44 +++--- onadata/apps/viewer/tests/test_export_list.py | 6 +- onadata/apps/viewer/tests/test_exports.py | 4 +- onadata/apps/viewer/tests/test_tasks.py | 6 +- .../test_instance_permission_filter.py | 12 +- .../serializers/test_attachment_serializer.py | 4 +- .../serializers/test_dataview_serializer.py | 4 +- .../serializers/test_metadata_serializer.py | 2 +- onadata/libs/tests/test_authentication.py | 4 +- onadata/libs/tests/test_renderers.py | 4 +- .../libs/tests/utils/test_api_export_tools.py | 2 +- onadata/libs/tests/utils/test_chart_tools.py | 4 +- onadata/libs/tests/utils/test_csv_builder.py | 6 +- onadata/libs/tests/utils/test_logger_tools.py | 122 ++++++++-------- onadata/libs/tests/utils/test_model_tools.py | 2 +- 42 files changed, 364 insertions(+), 364 deletions(-) diff --git a/onadata/apps/api/tests/models/test_project.py b/onadata/apps/api/tests/models/test_project.py index 2016fa2110..283476e89d 100644 --- a/onadata/apps/api/tests/models/test_project.py +++ b/onadata/apps/api/tests/models/test_project.py @@ -45,9 +45,9 @@ def test_project_soft_delete_works_when_no_exception_is_raised(self): project=project ) project.soft_delete() - self.assertEquals( + self.assertEqual( 1, Project.objects.filter(deleted_at__isnull=False).count()) - self.assertEquals( + self.assertEqual( 1, XForm.objects.filter(deleted_at__isnull=False).count()) def test_project_detetion_reverts_when_an_exception_raised(self): @@ -100,16 +100,16 @@ def test_project_detetion_reverts_when_an_exception_raised(self): with self.assertRaises(XLSFormError): project.soft_delete() - self.assertEquals(1, Project.objects.filter( + self.assertEqual(1, Project.objects.filter( deleted_at__isnull=True).count()) self.assertIsNone(project.deleted_at) - self.assertEquals(1, XForm.objects.filter( + self.assertEqual(1, XForm.objects.filter( project=project, deleted_at__isnull=True).count()) # Try deleting the Xform; it should also roll back due to the exception with self.assertRaises(XLSFormError): XForm.objects.all()[0].soft_delete() - self.assertEquals(1, XForm.objects.filter( + self.assertEqual(1, XForm.objects.filter( deleted_at__isnull=True).count()) self.assertIsNone(XForm.objects.all()[0].deleted_at) diff --git a/onadata/apps/api/tests/models/test_temp_token.py b/onadata/apps/api/tests/models/test_temp_token.py index 95bff92a88..8a262bf817 100644 --- a/onadata/apps/api/tests/models/test_temp_token.py +++ b/onadata/apps/api/tests/models/test_temp_token.py @@ -14,7 +14,7 @@ def test_temp_token_creation(self): self.assertEqual(token.key, '456c27c7d59303aed1dffd3b5ffaad36f0676618') self.assertTrue(created) - self.assertEquals(initial_count + 1, TempToken.objects.count()) + self.assertEqual(initial_count + 1, TempToken.objects.count()) # update initial count initial_count = TempToken.objects.count() @@ -35,7 +35,7 @@ def test_temp_token_creation(self): TempToken.objects.get( user=self.user, key='456c27c7d59303aed1dffd3b5ffaad36f0676618').delete() - self.assertEquals(initial_count - 1, TempToken.objects.count()) + self.assertEqual(initial_count - 1, TempToken.objects.count()) token1, created1 = TempToken.objects.get_or_create( user=self.user, key='1b3b82a23063a8a4e64fb4434dc21ab181fbbe7c') diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index 0780e6fe40..a0019cf8f5 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -764,7 +764,7 @@ def test_filter_pending_submission_reviews(self): # Confirm instance json now has _review_status field self.assertIn('_review_status', dict(instance.json)) # Confirm instance submission review status - self.assertEquals('1', instance.json['_review_status']) + self.assertEqual('1', instance.json['_review_status']) # Query xform data by submission review status 3 query_str = '{"_review_status": 3}' @@ -2026,10 +2026,10 @@ def test_geojson_geofield(self): response = view(request, pk=self.xform.pk, format='geojson') self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals( + self.assertEqual(response.data['type'], 'FeatureCollection') + self.assertEqual(len(response.data['features']), 4) + self.assertEqual(response.data['features'][0]['type'], 'Feature') + self.assertEqual( response.data['features'][0]['geometry']['geometries'][0]['type'], 'Point' ) @@ -2051,10 +2051,10 @@ def test_geojson_linestring(self): self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'Feature') - self.assertEquals(len(response.data['geometry']['coordinates']), 5) + self.assertEqual(response.data['type'], 'Feature') + self.assertEqual(len(response.data['geometry']['coordinates']), 5) self.assertIn('path', response.data['properties']) - self.assertEquals(response.data['geometry']['type'], 'LineString') + self.assertEqual(response.data['geometry']['type'], 'LineString') view = DataViewSet.as_view({'get': 'list'}) @@ -2062,10 +2062,10 @@ def test_geojson_linestring(self): response = view(request, pk=self.xform.pk, format='geojson') self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals(response.data['features'][0]['geometry']['type'], + self.assertEqual(response.data['type'], 'FeatureCollection') + self.assertEqual(len(response.data['features']), 4) + self.assertEqual(response.data['features'][0]['type'], 'Feature') + self.assertEqual(response.data['features'][0]['geometry']['type'], 'LineString') def test_geojson_polygon(self): @@ -2085,10 +2085,10 @@ def test_geojson_polygon(self): self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'Feature') - self.assertEquals(len(response.data['geometry']['coordinates'][0]), 6) + self.assertEqual(response.data['type'], 'Feature') + self.assertEqual(len(response.data['geometry']['coordinates'][0]), 6) self.assertIn('shape', response.data['properties']) - self.assertEquals(response.data['geometry']['type'], 'Polygon') + self.assertEqual(response.data['geometry']['type'], 'Polygon') view = DataViewSet.as_view({'get': 'list'}) @@ -2096,10 +2096,10 @@ def test_geojson_polygon(self): response = view(request, pk=self.xform.pk, format='geojson') self.assertEqual(response.status_code, 200) - self.assertEquals(response.data['type'], 'FeatureCollection') - self.assertEquals(len(response.data['features']), 4) - self.assertEquals(response.data['features'][0]['type'], 'Feature') - self.assertEquals(response.data['features'][0]['geometry']['type'], + self.assertEqual(response.data['type'], 'FeatureCollection') + self.assertEqual(len(response.data['features']), 4) + self.assertEqual(response.data['features'][0]['type'], 'Feature') + self.assertEqual(response.data['features'][0]['geometry']['type'], 'Polygon') def test_data_in_public_project(self): @@ -2110,7 +2110,7 @@ def test_data_in_public_project(self): formid = self.xform.pk with HTTMock(enketo_urls_mock): response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # get project id projectid = self.xform.project.pk @@ -2136,7 +2136,7 @@ def test_data_in_public_project(self): formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_diff_version(self): @@ -2162,19 +2162,19 @@ def test_data_diff_version(self): response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 5) + self.assertEqual(len(response.data), 5) query_str = '{"_version": "2014111"}' request = self.factory.get('/?query=%s' % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 4) + self.assertEqual(len(response.data), 4) query_str = '{"_version": "212121211"}' request = self.factory.get('/?query=%s' % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_last_modified_on_data_list_response(self): self._make_submissions() @@ -2184,7 +2184,7 @@ def test_last_modified_on_data_list_response(self): formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(response.get('Cache-Control'), 'max-age=60') self.assertTrue(response.has_header('ETag')) @@ -2195,9 +2195,9 @@ def test_last_modified_on_data_list_response(self): formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.assertEquals(etag_value, response.get('ETag')) + self.assertEqual(etag_value, response.get('ETag')) # delete one submission inst = Instance.objects.filter(xform=self.xform) @@ -2208,8 +2208,8 @@ def test_last_modified_on_data_list_response(self): formid = self.xform.pk response = view(request, pk=formid) - self.assertEquals(response.status_code, 200) - self.assertNotEquals(etag_value, response.get('ETag')) + self.assertEqual(response.status_code, 200) + self.assertNotEqual(etag_value, response.get('ETag')) def test_submission_history(self): """Test submission json includes has_history key""" diff --git a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py index 27ec249dfe..083026b8a5 100644 --- a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py @@ -554,14 +554,14 @@ def test_rest_service(self): request = self.factory.post('/', data=post_data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 3, RestService.objects.count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 3, RestService.objects.count()) # deleting the service for a merged xform deletes the same service from # the individual forms as well. service = RestService.objects.get(xform=xform) service.delete() - self.assertEquals(count, RestService.objects.count()) + self.assertEqual(count, RestService.objects.count()) def test_md_has_deleted_xforms(self): """ diff --git a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py index c13dad89a4..255c0690a7 100644 --- a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py @@ -102,7 +102,7 @@ def test_parse_error_is_raised(self): # Duplicate upload response = self._add_form_metadata(self.xform, data_type, self.data_value, self.path, False) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) self.assertIn(UNIQUE_TOGETHER_ERROR, response.data) def test_forms_endpoint_with_metadata(self): @@ -389,14 +389,14 @@ def test_should_return_both_xform_and_project_metadata(self): request = self.factory.get("/", **self.extra) response = view(request) - self.assertEquals(MetaData.objects.count(), expected_metadata_count) + self.assertEqual(MetaData.objects.count(), expected_metadata_count) for record in response.data: if record.get("xform"): - self.assertEquals(record.get('xform'), self.xform.id) + self.assertEqual(record.get('xform'), self.xform.id) self.assertIsNone(record.get('project')) else: - self.assertEquals(record.get('project'), self.project.id) + self.assertEqual(record.get('project'), self.project.id) self.assertIsNone(record.get('xform')) def test_should_only_return_xform_metadata(self): @@ -594,5 +594,5 @@ def test_unique_submission_review_metadata(self): request = self.factory.post('/', data, **self.extra) d_response = view(request) - self.assertEquals(d_response.status_code, 400) + self.assertEqual(d_response.status_code, 400) self.assertIn(UNIQUE_TOGETHER_ERROR, d_response.data) diff --git a/onadata/apps/api/tests/viewsets/test_ona_api.py b/onadata/apps/api/tests/viewsets/test_ona_api.py index 98a64f40ab..84f7e2b38e 100644 --- a/onadata/apps/api/tests/viewsets/test_ona_api.py +++ b/onadata/apps/api/tests/viewsets/test_ona_api.py @@ -18,7 +18,7 @@ def test_number_of_v1_viewsets(self): request = self.factory.get(path) request.resolver_match = resolve(path) response = view(request) - self.assertEquals(len(response.data), 29) + self.assertEqual(len(response.data), 29) def test_number_of_v2_viewsets(self): ''' @@ -30,4 +30,4 @@ def test_number_of_v2_viewsets(self): request = self.factory.get(path) request.resolver_match = resolve(path) response = view(request) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) diff --git a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py index 2ed4e777ba..6d145b0b73 100644 --- a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py @@ -239,7 +239,7 @@ def test_org_create_with_anonymous_user(self): '/', data=json.dumps(data), content_type="application/json") response = self.view(request) - self.assertEquals(response.status_code, 401) + self.assertEqual(response.status_code, 401) def test_orgs_members_list(self): self._org_create() @@ -409,7 +409,7 @@ def test_add_members_to_org_with_anonymous_user(self): response = view(request, user='denoinc') self.assertEqual(response.status_code, 401) - self.assertNotEquals(set(response.data), set([u'denoinc', u'aboy'])) + self.assertNotEqual(set(response.data), set([u'denoinc', u'aboy'])) def test_add_members_to_org_with_non_member_user(self): self._org_create() @@ -896,7 +896,7 @@ def test_org_always_has_admin_or_owner(self): response = view(request, user='denoinc') self.assertEqual(response.status_code, 400) - self.assertEquals(response.data, + self.assertEqual(response.data, { u'non_field_errors': [u'Organization cannot be without an owner'] @@ -966,7 +966,7 @@ def test_owner_not_allowed_to_be_removed(self): response = view(request, user='denoinc') self.assertEqual(response.status_code, 400) - self.assertEquals(response.data, + self.assertEqual(response.data, { u'non_field_errors': [u'Organization cannot be without an owner'] @@ -983,11 +983,11 @@ def test_orgs_delete(self): request = self.factory.delete('/', **self.extra) response = view(request, user='denoinc') - self.assertEquals(204, response.status_code) + self.assertEqual(204, response.status_code) - self.assertEquals(0, OrganizationProfile.objects.filter( + self.assertEqual(0, OrganizationProfile.objects.filter( user__username='denoinc').count()) - self.assertEquals(0, User.objects.filter(username='denoinc').count()) + self.assertEqual(0, User.objects.filter(username='denoinc').count()) def test_orgs_non_creator_delete(self): self._org_create() diff --git a/onadata/apps/api/tests/viewsets/test_osm_viewset.py b/onadata/apps/api/tests/viewsets/test_osm_viewset.py index 1f5388bd5a..026bff9d4e 100644 --- a/onadata/apps/api/tests/viewsets/test_osm_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_osm_viewset.py @@ -130,7 +130,7 @@ def test_osm_csv_export(self): response = view(request, pk=self.xform.pk, format='csv') self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers['Content-Type'], 'application/csv') diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index 0ab54e9258..82cbe92484 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -260,4 +260,4 @@ def test_wrong_stat_function_api(self): response = view(request, pk=formid) self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/api/tests/viewsets/test_team_viewset.py b/onadata/apps/api/tests/viewsets/test_team_viewset.py index 2505c42be9..c0e91c9f32 100644 --- a/onadata/apps/api/tests/viewsets/test_team_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_team_viewset.py @@ -282,7 +282,7 @@ def test_team_share_members(self): request = self.factory.get('/', data=get_data, **self.extra) response = view(request) # get the members team - self.assertEquals(response.data[1].get('name'), 'members') + self.assertEqual(response.data[1].get('name'), 'members') teamid = response.data[1].get('teamid') chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} diff --git a/onadata/apps/api/tests/viewsets/test_user_viewset.py b/onadata/apps/api/tests/viewsets/test_user_viewset.py index 7873ff73e0..57461f02ee 100644 --- a/onadata/apps/api/tests/viewsets/test_user_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_viewset.py @@ -91,7 +91,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params) response = view(request) - self.assertEquals(response.status_code, 401) + self.assertEqual(response.status_code, 401) error = {'detail': 'Authentication credentials were not provided.'} self.assertEqual(response.data, error) @@ -99,7 +99,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) get_params = { @@ -109,7 +109,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # empty results self.assertEqual(response.data, []) @@ -120,7 +120,7 @@ def test_get_user_using_email(self): request = self.factory.get('/', data=get_params, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) # empty results self.assertEqual(response.data, []) @@ -134,15 +134,15 @@ def test_get_non_org_users(self): all_users_request = self.factory.get('/') all_users_response = view(all_users_request) - self.assertEquals(all_users_response.status_code, 200) - self.assertEquals(len( + self.assertEqual(all_users_response.status_code, 200) + self.assertEqual(len( [u for u in all_users_response.data if u['username'] == 'denoinc'] ), 1) no_orgs_request = self.factory.get('/', data={'orgs': 'false'}) no_orgs_response = view(no_orgs_request) - self.assertEquals(no_orgs_response.status_code, 200) - self.assertEquals(len( + self.assertEqual(no_orgs_response.status_code, 200) + self.assertEqual(len( [u for u in no_orgs_response.data if u['username'] == 'denoinc']), 0) diff --git a/onadata/apps/api/tests/viewsets/test_widget_viewset.py b/onadata/apps/api/tests/viewsets/test_widget_viewset.py index 493502c9b6..0a7da9931d 100644 --- a/onadata/apps/api/tests/viewsets/test_widget_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_widget_viewset.py @@ -81,9 +81,9 @@ def test_create_using_unsupported_model_source(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals( + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual( response.data['content_object'], [u"`%s` is not a valid relation." % data['content_object']] ) @@ -102,9 +102,9 @@ def test_create_without_required_field(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['column'], + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual(response.data['column'], [u"This field is required."]) def test_create_unsupported_widget_type(self): @@ -122,9 +122,9 @@ def test_create_unsupported_widget_type(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['widget_type'], + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual(response.data['widget_type'], [u'"%s" is not a valid choice.' % data['widget_type']]) @@ -149,13 +149,13 @@ def test_update_widget(self): self.widget = Widget.objects.all().order_by('pk').reverse()[0] - self.assertEquals(key, self.widget.key) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'My new title updated') - self.assertEquals(response.data['key'], key) - self.assertEquals(response.data['description'], + self.assertEqual(key, self.widget.key) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['title'], 'My new title updated') + self.assertEqual(response.data['key'], key) + self.assertEqual(response.data['description'], "new description") - self.assertEquals(response.data['aggregation'], + self.assertEqual(response.data['aggregation'], "new aggregation") def test_patch_widget(self): @@ -168,8 +168,8 @@ def test_patch_widget(self): request = self.factory.patch('/', data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['column'], '_submitted_by') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['column'], '_submitted_by') def test_delete_widget(self): ct = ContentType.objects.get(model='xform', app_label='logger') @@ -180,11 +180,11 @@ def test_delete_widget(self): request = self.factory.delete('/', **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 204) + self.assertEqual(response.status_code, 204) after_count = Widget.objects.filter(content_type=ct, object_id=self.xform.pk).count() - self.assertEquals(count - 1, after_count) + self.assertEqual(count - 1, after_count) def test_list_widgets(self): self._create_widget() @@ -207,8 +207,8 @@ def test_list_widgets(self): request = self.factory.get('/', **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) def test_widget_permission_create(self): @@ -236,7 +236,7 @@ def test_widget_permission_create(self): content_type="application/json", **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) # owner OwnerRole.add(self.user, self.project) @@ -244,7 +244,7 @@ def test_widget_permission_create(self): content_type="application/json", **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # readonly ReadOnlyRole.add(self.user, self.project) @@ -252,7 +252,7 @@ def test_widget_permission_create(self): content_type="application/json", **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) # dataentryonlyrole DataEntryOnlyRole.add(self.user, self.project) @@ -261,7 +261,7 @@ def test_widget_permission_create(self): **self.extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_widget_permission_change(self): self._create_widget() @@ -278,8 +278,8 @@ def test_widget_permission_change(self): request = self.factory.patch('/', data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'Widget those') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['title'], 'Widget those') ReadOnlyRole.add(self.user, self.project) ReadOnlyRole.add(self.user, self.xform) @@ -287,8 +287,8 @@ def test_widget_permission_change(self): request = self.factory.patch('/', data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['title'], 'Widget those') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['title'], 'Widget those') def test_widget_permission_list(self): self._create_widget() @@ -303,8 +303,8 @@ def test_widget_permission_list(self): request = self.factory.get('/', **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 0) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 0) # assign alice the perms ReadOnlyRole.add(self.user, self.xform) @@ -312,8 +312,8 @@ def test_widget_permission_list(self): request = self.factory.get('/', **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) def test_widget_permission_get(self): self._create_widget() @@ -324,7 +324,7 @@ def test_widget_permission_get(self): request = self.factory.get('/', **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 404) + self.assertEqual(response.status_code, 404) # assign alice the perms ReadOnlyRole.add(self.user, self.project) @@ -333,7 +333,7 @@ def test_widget_permission_get(self): response = self.view(request, formid=self.xform.pk, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) def test_widget_data(self): self._create_widget() @@ -348,7 +348,7 @@ def test_widget_data(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) + self.assertEqual(len(response.data.get('data')['data']), 8) self.assertIn('age', response.data.get('data')['data'][0]) self.assertIn('count', response.data.get('data')['data'][0]) @@ -371,7 +371,7 @@ def test_widget_data_with_group_by(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 2) + self.assertEqual(len(response.data.get('data')['data']), 2) self.assertIn('gender', response.data.get('data')['data'][0]) self.assertIn('sum', response.data.get('data')['data'][0]) self.assertIn('mean', response.data.get('data')['data'][0]) @@ -395,7 +395,7 @@ def test_widget_data_widget(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) - self.assertEquals(response.data.get('data'), + self.assertEqual(response.data.get('data'), { 'field_type': u'select one', 'data_type': 'categorized', @@ -423,7 +423,7 @@ def test_widget_with_key(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) + self.assertEqual(len(response.data.get('data')['data']), 8) self.assertIn('age', response.data.get('data')['data'][0]) self.assertIn('count', response.data.get('data')['data'][0]) @@ -447,7 +447,7 @@ def test_widget_with_key_anon(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) self.assertIn('data', response.data.get('data')) - self.assertEquals(len(response.data.get('data')['data']), 8) + self.assertEqual(len(response.data.get('data')['data']), 8) self.assertIn('age', response.data.get('data')['data'][0]) self.assertIn('count', response.data.get('data')['data'][0]) @@ -481,7 +481,7 @@ def test_widget_data_public_form(self): response = view(request) self.assertEqual(response.status_code, 200) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) # Anonymous user can access widget in public form self.xform.shared_data = True @@ -491,7 +491,7 @@ def test_widget_data_public_form(self): response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_widget_pk_formid_required(self): self._create_widget() @@ -509,8 +509,8 @@ def test_widget_pk_formid_required(self): request = self.factory.put('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, {u'detail': u"'pk' required for this" u" action"}) @@ -539,8 +539,8 @@ def test_list_widgets_with_formid(self): request = self.factory.get('/', data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) def test_create_column_not_in_form(self): data = { @@ -556,9 +556,9 @@ def test_create_column_not_in_form(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(count, Widget.objects.all().count()) - self.assertEquals(response.data['column'], + self.assertEqual(response.status_code, 400) + self.assertEqual(count, Widget.objects.all().count()) + self.assertEqual(response.data['column'], [u"'doesnotexists' not in the form."]) def test_create_widget_with_xform_no_perms(self): @@ -576,8 +576,8 @@ def test_create_widget_with_xform_no_perms(self): request = self.factory.post('/', data=data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data['content_object'], + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data['content_object'], [u"You don't have permission to the Project."]) def test_filter_widgets_by_dataview(self): @@ -615,8 +615,8 @@ def test_filter_widgets_by_dataview(self): request = self.factory.get('/', data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 200) - self.assertEquals(len(response.data), 2) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) data = { "dataview": "so_invalid" @@ -625,8 +625,8 @@ def test_filter_widgets_by_dataview(self): request = self.factory.get('/', data=data, **self.extra) response = view(request) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data['detail'], + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data['detail'], u"Invalid value for dataview %s." % "so_invalid") def test_order_widget(self): @@ -642,14 +642,14 @@ def test_order_widget(self): request = self.factory.patch('/', data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['order'], 1) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['order'], 1) widget = Widget.objects.all().order_by('pk')[0] - self.assertEquals(widget.order, 0) + self.assertEqual(widget.order, 0) widget = Widget.objects.all().order_by('pk')[1] - self.assertEquals(widget.order, 2) + self.assertEqual(widget.order, 2) def test_widget_data_case_sensitive(self): xlsform_path = os.path.join( @@ -686,7 +686,7 @@ def test_widget_data_case_sensitive(self): self.assertEqual(response.status_code, 200) self.assertIsNotNone(response.data.get('data')) - self.assertEquals(response.data.get('data'), + self.assertEqual(response.data.get('data'), { 'field_type': u'select one', 'data_type': 'categorized', @@ -740,7 +740,7 @@ def test_widget_create_by_org_admin(self): **extra) response = view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) def test_create_multiple_choice(self): data = { diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 4ca8c45f8f..b7be15519a 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -1027,9 +1027,9 @@ def test_enketo_urls_remain_the_same_after_form_replacement(self): self.xform.refresh_from_db() # diff versions - self.assertNotEquals(version, self.xform.version) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(version, self.xform.version) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) def test_xform_hash_changes_after_form_replacement(self): with HTTMock(enketo_mock): @@ -1061,7 +1061,7 @@ def test_xform_hash_changes_after_form_replacement(self): self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - self.assertNotEquals(xform_old_hash, self.xform.hash) + self.assertNotEqual(xform_old_hash, self.xform.hash) def test_hash_changes_after_update_xform_xls_file(self): with HTTMock(enketo_mock): @@ -1092,7 +1092,7 @@ def test_hash_changes_after_update_xform_xls_file(self): self.assertEqual(response.status_code, 200) self.xform.refresh_from_db() - self.assertNotEquals(xform_old_hash, self.xform.hash) + self.assertNotEqual(xform_old_hash, self.xform.hash) def test_login_enketo_no_redirect(self): with HTTMock(enketo_urls_mock): @@ -1242,7 +1242,7 @@ def test_publish_xlsform(self): self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1298,7 +1298,7 @@ def test_publish_xlsforms_with_same_id_string(self): ) self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1339,7 +1339,7 @@ def test_publish_xlsforms_with_same_id_string(self): self.assertDictContainsSubset(data, response.data) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"]) + self.assertEqual("owner", response.data["users"][0]["role"]) # pylint: disable=no-member self.assertIsNotNone( @@ -1528,7 +1528,7 @@ def test_publish_select_external_xlsform(self): ) self.assertIsNotNone(metadata) self.assertTrue(OwnerRole.user_has_role(self.user, xform)) - self.assertEquals("owner", response.data["users"][0]["role"], self.user) + self.assertEqual("owner", response.data["users"][0]["role"], self.user) def test_publish_csv_with_universal_newline_xlsform(self): with HTTMock(enketo_mock): @@ -2099,7 +2099,7 @@ def test_external_export(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) def test_external_export_with_data_id(self): with HTTMock(enketo_mock): @@ -2141,7 +2141,7 @@ def test_external_export_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) def test_external_export_error(self): with HTTMock(enketo_mock): @@ -2317,7 +2317,7 @@ def test_csv_import_diff_column(self): response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 400) self.assertIn("error", response.data) - self.assertEquals( + self.assertEqual( response.data.get("error"), "Sorry uploaded file does not match the form. " "The file is missing the column(s): age, name.", @@ -2342,7 +2342,7 @@ def test_csv_import_additional_columns(self): self.assertEqual(response.status_code, 200) self.assertIn("info", response.data) - self.assertEquals( + self.assertEqual( response.data.get("info"), "Additional column(s) excluded from the upload:" " '_additional'.", ) @@ -2475,11 +2475,11 @@ def test_update_xform_xls_file(self): new_version = self.xform.version new_last_updated_at = self.xform.last_updated_at # diff versions - self.assertNotEquals(last_updated_at, new_last_updated_at) - self.assertNotEquals(version, new_version) - self.assertNotEquals(title_old, self.xform.title) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(last_updated_at, new_last_updated_at) + self.assertNotEqual(version, new_version) + self.assertNotEqual(title_old, self.xform.title) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) def test_manager_can_update_xform_xls_file(self): """Manager Role can replace xlsform""" @@ -2531,10 +2531,10 @@ def test_manager_can_update_xform_xls_file(self): new_version = self.xform.version # diff versions - self.assertNotEquals(version, new_version) - self.assertNotEquals(title_old, self.xform.title) - self.assertEquals(form_id, self.xform.pk) - self.assertEquals(id_string, self.xform.id_string) + self.assertNotEqual(version, new_version) + self.assertNotEqual(title_old, self.xform.title) + self.assertEqual(form_id, self.xform.pk) + self.assertEqual(id_string, self.xform.id_string) xml = self.xform.xml fhuuid = xml.find("formhub/uuid") self.assertEqual(xml[xml[:fhuuid].rfind("=") + 2 : fhuuid], "/data/") @@ -2797,8 +2797,8 @@ def test_update_xform_xls_bad_file(self): new_version = self.xform.version # fails to update the form - self.assertEquals(version, new_version) - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(version, new_version) + self.assertEqual(form_id, self.xform.pk) def test_update_xform_xls_file_with_submissions(self): with HTTMock(enketo_mock): @@ -2836,10 +2836,10 @@ def test_update_xform_xls_file_with_submissions(self): self.xform.refresh_from_db() - self.assertEquals(form_id, self.xform.pk) - self.assertNotEquals(version, self.xform.version) - self.assertNotEquals(xform_json, self.xform.json) - self.assertNotEquals(xform_xml, self.xform.xml) + self.assertEqual(form_id, self.xform.pk) + self.assertNotEqual(version, self.xform.version) + self.assertNotEqual(xform_json, self.xform.json) + self.assertNotEqual(xform_xml, self.xform.xml) is_updated_form = ( len( [ @@ -2884,8 +2884,8 @@ def test_update_xform_xls_file_with_version_set(self): self.xform.refresh_from_db() # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) @patch("onadata.apps.main.forms.urlopen") def test_update_xform_xls_url(self, mock_urlopen): @@ -2924,10 +2924,10 @@ def test_update_xform_xls_url(self, mock_urlopen): self.xform.refresh_from_db() - self.assertEquals(count, XForm.objects.all().count()) + self.assertEqual(count, XForm.objects.all().count()) # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) @patch("onadata.apps.main.forms.urlopen") def test_update_xform_dropbox_url(self, mock_urlopen): @@ -2966,10 +2966,10 @@ def test_update_xform_dropbox_url(self, mock_urlopen): self.xform.refresh_from_db() - self.assertEquals(count, XForm.objects.all().count()) + self.assertEqual(count, XForm.objects.all().count()) # diff versions - self.assertEquals(self.xform.version, "212121211") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) def test_update_xform_using_put_with_invalid_input(self): with HTTMock(enketo_mock): @@ -3073,7 +3073,7 @@ def test_update_xform_using_put_with_invalid_input(self): self.xform.refresh_from_db() # check that description has been sanitized - self.assertEquals( + self.assertEqual( conditional_escape(unsanitized_html_str), self.xform.description ) @@ -3109,10 +3109,10 @@ def test_update_xform_using_put(self): self.xform.refresh_from_db() - self.assertEquals(version, self.xform.version) - self.assertEquals(self.xform.description, "Transport form") - self.assertEquals(self.xform.title, "Transport Form") - self.assertEquals(form_id, self.xform.pk) + self.assertEqual(version, self.xform.version) + self.assertEqual(self.xform.description, "Transport form") + self.assertEqual(self.xform.title, "Transport Form") + self.assertEqual(form_id, self.xform.pk) def test_update_xform_using_put_without_required_field(self): with HTTMock(enketo_mock): @@ -3142,7 +3142,7 @@ def test_update_xform_using_put_without_required_field(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.get("Cache-Control"), None) - self.assertEquals(response.data, {"title": ["This field is required."]}) + self.assertEqual(response.data, {"title": ["This field is required."]}) def test_public_xform_accessible_by_authenticated_users(self): with HTTMock(enketo_mock): @@ -3184,7 +3184,7 @@ def test_publish_form_async(self, mock_get_status): self.assertTrue("job_uuid" in response.data) - self.assertEquals(count + 1, XForm.objects.count()) + self.assertEqual(count + 1, XForm.objects.count()) # get the result get_data = {"job_uuid": response.data.get("job_uuid")} @@ -3194,7 +3194,7 @@ def test_publish_form_async(self, mock_get_status): self.assertTrue(mock_get_status.called) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"job_status": "PENDING"}) + self.assertEqual(response.data, {"job_status": "PENDING"}) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch( @@ -3284,7 +3284,7 @@ def test_failed_form_publishing_after_maximum_retries( response = view(request) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, error_message) + self.assertEqual(response.data, error_message) def test_survey_preview_endpoint(self): view = XFormViewSet.as_view({"post": "survey_preview", "get": "survey_preview"}) @@ -3366,7 +3366,7 @@ def test_delete_xform_async(self, mock_get_status): self.assertEqual(response.status_code, 202) self.assertTrue("job_uuid" in response.data) self.assertTrue("time_async_triggered" in response.data) - self.assertEquals(count, XForm.objects.count()) + self.assertEqual(count, XForm.objects.count()) view = XFormViewSet.as_view({"get": "delete_async"}) @@ -3376,7 +3376,7 @@ def test_delete_xform_async(self, mock_get_status): self.assertTrue(mock_get_status.called) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"job_status": "PENDING"}) + self.assertEqual(response.data, {"job_status": "PENDING"}) xform = XForm.objects.get(pk=formid) @@ -3636,7 +3636,7 @@ def test_check_async_publish_empty_uuid(self): response = view(request) self.assertEqual(response.status_code, 202) - self.assertEquals(response.data, {"error": "Empty job uuid"}) + self.assertEqual(response.data, {"error": "Empty job uuid"}) def test_always_new_report_with_data_id(self): with HTTMock(enketo_mock): @@ -3679,7 +3679,7 @@ def test_always_new_report_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc059" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) count = Export.objects.filter( xform=self.xform, export_type=Export.EXTERNAL_EXPORT @@ -3690,13 +3690,13 @@ def test_always_new_report_with_data_id(self): response = view(request, pk=formid, format="xls") self.assertEqual(response.status_code, 302) expected_url = "http://xls_server/xls/ee3ff9d8f5184fc4a8fdebc2547cc057" - self.assertEquals(response.url, expected_url) + self.assertEqual(response.url, expected_url) count2 = Export.objects.filter( xform=self.xform, export_type=Export.EXTERNAL_EXPORT ).count() - self.assertEquals(count + 1, count2) + self.assertEqual(count + 1, count2) def test_different_form_versions(self): with HTTMock(enketo_mock): @@ -4249,7 +4249,7 @@ def test_csv_export_no_new_generated(self): response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers["Content-Type"], "application/csv") @@ -4264,7 +4264,7 @@ def test_csv_export_no_new_generated(self): self.assertEqual(response.status_code, 200) # no new export generated - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) self.assertEqual(headers["Content-Type"], "application/csv") @@ -4361,7 +4361,7 @@ def test_xform_linked_dataviews(self): self.assertEqual(response.status_code, 200) self.assertIn("data_views", response.data) - self.assertEquals(2, len(response.data["data_views"])) + self.assertEqual(2, len(response.data["data_views"])) def test_delete_xform_also_deletes_linked_dataviews(self): """ @@ -4410,7 +4410,7 @@ def test_delete_xform_also_deletes_linked_dataviews(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIn("data_views", response.data) - self.assertEquals(2, len(response.data["data_views"])) + self.assertEqual(2, len(response.data["data_views"])) # delete xform view = XFormViewSet.as_view({"delete": "destroy", "get": "retrieve"}) @@ -4445,7 +4445,7 @@ def test_multitple_enketo_urls(self): count = MetaData.objects.filter( object_id=self.xform.id, data_type="enketo_url" ).count() - self.assertEquals(2, count) + self.assertEqual(2, count) # delete cache safe_delete(f"{ENKETO_URL_CACHE}{self.xform.pk}") @@ -4531,7 +4531,7 @@ def test_csv_export_filtered_by_date(self): count = Export.objects.all().count() response = self._get_date_filtered_export(query_str) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) test_file_path = os.path.join( settings.PROJECT_ROOT, @@ -4549,7 +4549,7 @@ def test_csv_export_filtered_by_date(self): export = Export.objects.last() self.assertIn("query", export.options) - self.assertEquals(export.options["query"], query_str) + self.assertEqual(export.options["query"], query_str) @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_form_data_async_with_filtered_date(self, async_result): @@ -4585,7 +4585,7 @@ def test_export_form_data_async_with_filtered_date(self, async_result): self.assertIsNotNone(response.data) self.assertEqual(response.status_code, 202) self.assertTrue("job_uuid" in response.data) - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) task_id = response.data.get("job_uuid") get_data = {"job_uuid": task_id} @@ -4599,7 +4599,7 @@ def test_export_form_data_async_with_filtered_date(self, async_result): export = Export.objects.last() self.assertIn("query", export.options) - self.assertEquals(export.options["query"], query_str) + self.assertEqual(export.options["query"], query_str) def test_previous_export_with_date_filter_is_returned(self): with HTTMock(enketo_mock): @@ -4626,7 +4626,7 @@ def test_previous_export_with_date_filter_is_returned(self): self._get_date_filtered_export(query_str) # no change in count of exports - self.assertEquals(count, Export.objects.all().count()) + self.assertEqual(count, Export.objects.all().count()) def test_download_export_with_export_id(self): with HTTMock(enketo_mock): @@ -4710,7 +4710,7 @@ def test_normal_export_after_export_with_date_filter(self): self.assertEqual(response.status_code, 200) # should create a new export - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_form_data_async_include_labels(self, async_result): @@ -5203,7 +5203,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # should generate new - self.assertEquals(count + 1, Export.objects.all().count()) + self.assertEqual(count + 1, Export.objects.all().count()) survey = self.surveys[0] self._make_submission( @@ -5227,7 +5227,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # changed options, should generate new - self.assertEquals(count + 2, Export.objects.all().count()) + self.assertEqual(count + 2, Export.objects.all().count()) data = {"export_type": "csv", "win_excel_utf8": False} @@ -5236,7 +5236,7 @@ def test_csv_export_cache(self): self.assertEqual(response.status_code, 200) # reused options, should generate new with new submission - self.assertEquals(count + 3, Export.objects.all().count()) + self.assertEqual(count + 3, Export.objects.all().count()) def test_upload_xml_form_file(self): with HTTMock(enketo_mock): diff --git a/onadata/apps/logger/tests/models/test_data_view.py b/onadata/apps/logger/tests/models/test_data_view.py index e63d25ac9f..c8293ee158 100644 --- a/onadata/apps/logger/tests/models/test_data_view.py +++ b/onadata/apps/logger/tests/models/test_data_view.py @@ -91,13 +91,13 @@ def test_generate_query_string_for_data_without_filter(self): self.all_data, self.sort) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) self.assertEqual(len(columns), 8) self.cursor.execute(sql, [str(i) for i in (params)]) results = self.cursor.fetchall() - self.assertEquals(len(results), 3) + self.assertEqual(len(results), 3) def test_generate_query_string_for_data_with_limit_filter(self): limit_filter = 1 @@ -114,14 +114,14 @@ def test_generate_query_string_for_data_with_limit_filter(self): self.all_data, self.sort) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) records = [record for record in DataView.query_iterator(sql, columns, params, self.count)] - self.assertEquals(len(records), limit_filter) + self.assertEqual(len(records), limit_filter) def test_generate_query_string_for_data_with_start_index_filter(self): start_index = 2 @@ -138,13 +138,13 @@ def test_generate_query_string_for_data_with_start_index_filter(self): self.all_data, self.sort) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) records = [record for record in DataView.query_iterator(sql, columns, params, self.count)] - self.assertEquals(len(records), 1) + self.assertEqual(len(records), 1) self.assertIn('name', records[0]) self.assertIn('age', records[0]) self.assertIn('gender', records[0]) @@ -165,7 +165,7 @@ def test_generate_query_string_for_data_with_sort_column_asc(self): self.all_data, sort) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) records = [record for record in DataView.query_iterator(sql, columns, @@ -189,7 +189,7 @@ def test_generate_query_string_for_data_with_sort_column_desc(self): self.all_data, sort) - self.assertEquals(sql, expected_sql) + self.assertEqual(sql, expected_sql) records = [record for record in DataView.query_iterator(sql, columns, diff --git a/onadata/apps/logger/tests/test_form_list.py b/onadata/apps/logger/tests/test_form_list.py index 4009f196bd..a2a01cd19f 100644 --- a/onadata/apps/logger/tests/test_form_list.py +++ b/onadata/apps/logger/tests/test_form_list.py @@ -41,4 +41,4 @@ def test_show_for_anon_when_require_auth_false(self): request = self.factory.get('/') request.user = AnonymousUser() response = formList(request, self.user.username) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/tests/test_form_submission.py b/onadata/apps/logger/tests/test_form_submission.py index 9d92014751..b18eb69bcb 100644 --- a/onadata/apps/logger/tests/test_form_submission.py +++ b/onadata/apps/logger/tests/test_form_submission.py @@ -84,7 +84,7 @@ def test_fail_with_ioerror_read(self, mock_pop): mock_pop.side_effect = IOError( 'request data read error') - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -93,14 +93,14 @@ def test_fail_with_ioerror_read(self, mock_pop): self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) @patch('django.utils.datastructures.MultiValueDict.pop') def test_fail_with_ioerror_wsgi(self, mock_pop): mock_pop.side_effect = IOError( 'error during read(65536) on wsgi.input') - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -109,7 +109,7 @@ def test_fail_with_ioerror_wsgi(self, mock_pop): self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) def test_submission_to_require_auth_anon(self): """ @@ -349,7 +349,7 @@ def test_edited_submission(self): # check that instance history's submission_date is equal to instance's # date_created - last_edited by default is null for an instance - self.assertEquals(edited_instance.date_created, + self.assertEqual(edited_instance.date_created, instance_history_1.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) @@ -376,7 +376,7 @@ def test_edited_submission(self): record = cursor[0] edited_instance = self.xform.instances.first() instance_history_2 = InstanceHistory.objects.last() - self.assertEquals(instance_before_second_edit.last_edited, + self.assertEqual(instance_before_second_edit.last_edited, instance_history_2.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) @@ -586,7 +586,7 @@ def test_fail_with_unreadable_post_error(self, mock_create_instance): 'error during read(65536) on wsgi.input' ) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -595,7 +595,7 @@ def test_fail_with_unreadable_post_error(self, mock_create_instance): self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) - self.assertEquals(0, self.xform.instances.count()) + self.assertEqual(0, self.xform.instances.count()) def test_form_submission_with_infinity_values(self): """ @@ -616,6 +616,6 @@ def test_form_submission_with_infinity_values(self): ) self._make_submission(path=xml_submission_file_path) - self.assertEquals(400, self.response.status_code) + self.assertEqual(400, self.response.status_code) self.assertIn( 'invalid input syntax for type json', str(self.response.message)) diff --git a/onadata/apps/logger/tests/test_parsing.py b/onadata/apps/logger/tests/test_parsing.py index 60928091a4..7cc1d60639 100644 --- a/onadata/apps/logger/tests/test_parsing.py +++ b/onadata/apps/logger/tests/test_parsing.py @@ -173,7 +173,7 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): "multiple_nodes_error.xml" ) self._make_submission(xml_submission_file_path) - self.assertEquals(201, self.response.status_code) + self.assertEqual(201, self.response.status_code) def test_multiple_media_files_on_encrypted_form(self): self._create_user_and_login() diff --git a/onadata/apps/logger/tests/test_simple_submission.py b/onadata/apps/logger/tests/test_simple_submission.py index 5c060e8ae0..b045eff26e 100644 --- a/onadata/apps/logger/tests/test_simple_submission.py +++ b/onadata/apps/logger/tests/test_simple_submission.py @@ -82,31 +82,31 @@ def test_start_time_boolean_properly_set(self): self.assertTrue(self.xform2.has_start_time) def test_simple_yes_submission(self): - self.assertEquals(0, self.xform1.instances.count()) + self.assertEqual(0, self.xform1.instances.count()) self._submit_simple_yes() - self.assertEquals(1, self.xform1.instances.count()) + self.assertEqual(1, self.xform1.instances.count()) self._submit_simple_yes() # a simple "yes" submission *SHOULD* increment the survey count - self.assertEquals(2, self.xform1.instances.count()) + self.assertEqual(2, self.xform1.instances.count()) def test_start_time_submissions(self): """This test checks to make sure that instances *with start_time available* are marked as duplicates when the XML is a direct match. """ - self.assertEquals(0, self.xform2.instances.count()) + self.assertEqual(0, self.xform2.instances.count()) self._submit_at_hour(11) - self.assertEquals(1, self.xform2.instances.count()) + self.assertEqual(1, self.xform2.instances.count()) self._submit_at_hour(12) - self.assertEquals(2, self.xform2.instances.count()) + self.assertEqual(2, self.xform2.instances.count()) # an instance from 11 AM already exists in the database, so it # *SHOULD NOT* increment the survey count. self._submit_at_hour(11) - self.assertEquals(2, self.xform2.instances.count()) + self.assertEqual(2, self.xform2.instances.count()) def test_corrupted_submission(self): """Test xml submissions that contain unicode characters. diff --git a/onadata/apps/main/tests/test_csv_export.py b/onadata/apps/main/tests/test_csv_export.py index f7b022fc7e..db1ca74bb1 100644 --- a/onadata/apps/main/tests/test_csv_export.py +++ b/onadata/apps/main/tests/test_csv_export.py @@ -73,7 +73,7 @@ def test_csv_nested_repeat_output(self): u'/data/bed_net[2]/member[2]/name', u'/data/meta/instanceID' ] - self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths) + self.assertEqual(data_dictionary.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export( Export.CSV_EXPORT, diff --git a/onadata/apps/main/tests/test_form_auth.py b/onadata/apps/main/tests/test_form_auth.py index 45351c6195..3167918098 100644 --- a/onadata/apps/main/tests/test_form_auth.py +++ b/onadata/apps/main/tests/test_form_auth.py @@ -11,4 +11,4 @@ def setUp(self): def test_login_redirect_redirects(self): response = self.client.get(reverse(login_redirect)) - self.assertEquals(response.status_code, 302) + self.assertEqual(response.status_code, 302) diff --git a/onadata/apps/main/tests/test_form_errors.py b/onadata/apps/main/tests/test_form_errors.py index d4992f72b1..77bbcf7003 100644 --- a/onadata/apps/main/tests/test_form_errors.py +++ b/onadata/apps/main/tests/test_form_errors.py @@ -28,7 +28,7 @@ def test_bad_id_string(self): xls_path = os.path.join(self.this_directory, "fixtures", "transportation", "transportation.bad_id.xlsx") self.assertRaises(XLSFormError, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) @skip def test_dl_no_xls(self): @@ -99,7 +99,7 @@ def test_spaced_xlsform(self): " have modified the filename to not contain any spaces.") self.assertRaisesMessage( XLSFormError, msg, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) def test_choice_duplicate_error(self): """ @@ -122,4 +122,4 @@ def test_choice_duplicate_error(self): "Learn more: https://xlsform.org/#choice-names.") self.assertRaisesMessage( PyXFormError, msg, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), count) + self.assertEqual(XForm.objects.count(), count) diff --git a/onadata/apps/main/tests/test_form_metadata.py b/onadata/apps/main/tests/test_form_metadata.py index 2eec07e1b1..c7012e1a62 100644 --- a/onadata/apps/main/tests/test_form_metadata.py +++ b/onadata/apps/main/tests/test_form_metadata.py @@ -88,7 +88,7 @@ def test_adds_supporting_doc_on_submit(self): count = len(MetaData.objects.filter(object_id=self.xform.id, data_type='supporting_doc')) self._add_metadata() - self.assertEquals(count + 1, len(MetaData.objects.filter( + self.assertEqual(count + 1, len(MetaData.objects.filter( object_id=self.xform.id, data_type='supporting_doc'))) def test_delete_cached_xform_metadata_object_on_save(self): @@ -98,13 +98,13 @@ def test_delete_cached_xform_metadata_object_on_save(self): self.assertIsNone( cache.get('{}{}'.format(XFORM_METADATA_CACHE, self.xform.id))) - self.assertEquals(count + 1, MetaData.objects.count()) + self.assertEqual(count + 1, MetaData.objects.count()) def test_adds_supporting_media_on_submit(self): count = len(MetaData.objects.filter(object_id=self.xform.id, data_type='media')) self._add_metadata(data_type='media') - self.assertEquals(count + 1, len(MetaData.objects.filter( + self.assertEqual(count + 1, len(MetaData.objects.filter( object_id=self.xform.id, data_type='media'))) def test_adds_mapbox_layer_on_submit(self): @@ -114,7 +114,7 @@ def test_adds_mapbox_layer_on_submit(self): self.post_data['map_name'] = 'test_mapbox_layer' self.post_data['link'] = 'http://0.0.0.0:8080' self.client.post(self.edit_url, self.post_data) - self.assertEquals(count + 1, MetaData.objects.filter( + self.assertEqual(count + 1, MetaData.objects.filter( object_id=self.xform.id, data_type='mapbox_layer').count()) def test_shows_supporting_doc_after_submit(self): @@ -305,7 +305,7 @@ def test_add_media_url(self): uri = 'https://devtrac.ona.io/fieldtrips.csv' count = MetaData.objects.filter(data_type='media').count() self.client.post(self.edit_url, {'media_url': uri}) - self.assertEquals(count + 1, + self.assertEqual(count + 1, len(MetaData.objects.filter(data_type='media'))) def test_windows_csv_file_upload(self): diff --git a/onadata/apps/main/tests/test_metadata.py b/onadata/apps/main/tests/test_metadata.py index f119a18fb6..3e37a5ec4f 100644 --- a/onadata/apps/main/tests/test_metadata.py +++ b/onadata/apps/main/tests/test_metadata.py @@ -21,7 +21,7 @@ def test_create_metadata(self): data_type='enketo_url')) enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) - self.assertEquals(count + 1, len(MetaData.objects.filter( + self.assertEqual(count + 1, len(MetaData.objects.filter( object_id=self.xform.id, data_type='enketo_url'))) def test_create_google_sheet_metadata_object(self): @@ -34,7 +34,7 @@ def test_create_google_sheet_metadata_object(self): ).format(GOOGLE_SHEET_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, USER_ID) MetaData.set_google_sheet_details(self.xform, google_sheets_actions) # change - self.assertEquals(count + 1, MetaData.objects.filter( + self.assertEqual(count + 1, MetaData.objects.filter( object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE).count()) gsheet_details = MetaData.get_google_sheet_details(self.xform.pk) @@ -49,11 +49,11 @@ def test_saving_same_metadata_object_doesnt_trigger_integrity_error(self): enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) count += 1 - self.assertEquals(count, len(MetaData.objects.filter( + self.assertEqual(count, len(MetaData.objects.filter( object_id=self.xform.id, data_type='enketo_url'))) MetaData.enketo_url(self.xform, enketo_url) - self.assertEquals(count, len(MetaData.objects.filter( + self.assertEqual(count, len(MetaData.objects.filter( object_id=self.xform.id, data_type='enketo_url'))) def test_unique_type_for_form(self): @@ -77,7 +77,7 @@ def test_upload_to_with_anonymous_user(self): metadata = MetaData(data_type="media") metadata.content_object = instance filename = "filename" - self.assertEquals(upload_to(metadata, filename), + self.assertEqual(upload_to(metadata, filename), "{}/{}/{}".format(self.user.username, 'formid-media', filename)) @@ -85,7 +85,7 @@ def test_upload_to_with_anonymous_user(self): instance_without_user = Instance(xform=self.xform) metadata.content_object = instance_without_user - self.assertEquals(upload_to(metadata, filename), + self.assertEqual(upload_to(metadata, filename), "{}/{}/{}".format(self.xform.user.username, 'formid-media', filename)) @@ -97,7 +97,7 @@ def test_upload_to_with_project_and_xform_instance(self): filename = "filename" - self.assertEquals(upload_to(metadata, filename), + self.assertEqual(upload_to(metadata, filename), "{}/{}/{}".format(self.user.username, 'formid-media', filename)) @@ -107,7 +107,7 @@ def test_upload_to_with_project_and_xform_instance(self): filename = "filename" - self.assertEquals(upload_to(metadata, filename), + self.assertEqual(upload_to(metadata, filename), "{}/{}/{}".format(self.user.username, 'formid-media', filename)) diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 9bfcafc3c2..0ac3388ffe 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -365,7 +365,7 @@ def _check_data_for_csv_export(self): test_dict[new_key] = d_from_db[k] self.assertTrue(test_dict in data, (test_dict, data)) data.remove(test_dict) - self.assertEquals(data, []) + self.assertEqual(data, []) def _check_group_xpaths_do_not_appear_in_dicts_for_export(self): uuid = "uuid:f3d8dc65-91a6-4d0f-9e97-802128083390" @@ -554,9 +554,9 @@ def _check_xls_export(self): self.assertEqual(actual_row, expected_row) def _check_delete(self): - self.assertEquals(self.user.xforms.count(), 1) + self.assertEqual(self.user.xforms.count(), 1) self.user.xforms.all()[0].delete() - self.assertEquals(self.user.xforms.count(), 0) + self.assertEqual(self.user.xforms.count(), 0) def test_405_submission(self): url = reverse("submissions") diff --git a/onadata/apps/main/tests/test_user_id_string_unique_together.py b/onadata/apps/main/tests/test_user_id_string_unique_together.py index 3f0d0e1fe9..aec4c9ed8e 100644 --- a/onadata/apps/main/tests/test_user_id_string_unique_together.py +++ b/onadata/apps/main/tests/test_user_id_string_unique_together.py @@ -19,14 +19,14 @@ def test_unique_together(self): # first time self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), 1) + self.assertEqual(XForm.objects.count(), 1) # second time self.assertRaises(IntegrityError, self._publish_xls_file, xls_path) - self.assertEquals(XForm.objects.count(), 1) + self.assertEqual(XForm.objects.count(), 1) self.client.logout() # first time self._create_user_and_login(username="carl", password="carl") self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), 2) + self.assertEqual(XForm.objects.count(), 2) diff --git a/onadata/apps/messaging/tests/test_backends_mqtt.py b/onadata/apps/messaging/tests/test_backends_mqtt.py index 3106379977..e4d3d3176e 100644 --- a/onadata/apps/messaging/tests/test_backends_mqtt.py +++ b/onadata/apps/messaging/tests/test_backends_mqtt.py @@ -134,12 +134,12 @@ def test_mqtt_send(self, mocked): mqtt.send(instance=instance) self.assertTrue(mocked.called) args, kwargs = mocked.call_args_list[0] - self.assertEquals(mqtt.get_topic(instance), args[0]) - self.assertEquals(get_payload(instance), kwargs['payload']) - self.assertEquals('localhost', kwargs['hostname']) - self.assertEquals(8883, kwargs['port']) - self.assertEquals(0, kwargs['qos']) - self.assertEquals(False, kwargs['retain']) + self.assertEqual(mqtt.get_topic(instance), args[0]) + self.assertEqual(get_payload(instance), kwargs['payload']) + self.assertEqual('localhost', kwargs['hostname']) + self.assertEqual(8883, kwargs['port']) + self.assertEqual(0, kwargs['qos']) + self.assertEqual(False, kwargs['retain']) self.assertDictEqual( dict(ca_certs='cacert.pem', certfile='emq.pem', diff --git a/onadata/apps/restservice/tests/test_restservice.py b/onadata/apps/restservice/tests/test_restservice.py index a531d2577a..3a4677e26e 100644 --- a/onadata/apps/restservice/tests/test_restservice.py +++ b/onadata/apps/restservice/tests/test_restservice.py @@ -48,7 +48,7 @@ def _add_rest_service(self, service_url, service_name): 'service_name': service_name} response = self.client.post(add_service_url, post_data) self.assertEqual(response.status_code, 200) - self.assertEquals(RestService.objects.all().count(), count + 1) + self.assertEqual(RestService.objects.all().count(), count + 1) def add_rest_service_with_usename_and_id_string_in_uppercase(self): add_service_url = reverse(add_service, kwargs={ @@ -61,7 +61,7 @@ def add_rest_service_with_usename_and_id_string_in_uppercase(self): def test_create_rest_service(self): count = RestService.objects.all().count() self._create_rest_service() - self.assertEquals(RestService.objects.all().count(), count + 1) + self.assertEqual(RestService.objects.all().count(), count + 1) def test_service_definition(self): self._create_rest_service() @@ -134,7 +134,7 @@ def test_textit_service(self, mock_http): self.assertFalse(mock_http.called) self._make_submission(xml_submission) self.assertTrue(mock_http.called) - self.assertEquals(mock_http.call_count, 1) + self.assertEqual(mock_http.call_count, 1) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch('requests.post') @@ -146,7 +146,7 @@ def test_rest_service_not_set(self, mock_http): self.assertFalse(mock_http.called) self._make_submission(xml_submission) self.assertFalse(mock_http.called) - self.assertEquals(mock_http.call_count, 0) + self.assertEqual(mock_http.call_count, 0) def test_clean_keys_of_slashes(self): service = ServiceDefinition() @@ -164,5 +164,5 @@ def test_clean_keys_of_slashes(self): "zero_column": "0" } - self.assertEquals(expected_data, + self.assertEqual(expected_data, service.clean_keys_of_slashes(test_data)) diff --git a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py index 0fd8b1536f..189bcc70a5 100644 --- a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py +++ b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py @@ -33,8 +33,8 @@ def test_create(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, RestService.objects.all().count()) def test_textit_service_missing_params(self): post_data = { @@ -45,7 +45,7 @@ def test_textit_service_missing_params(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) def _create_textit_service(self): count = RestService.objects.all().count() @@ -61,12 +61,12 @@ def _create_textit_service(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) - self.assertEquals(count + 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 201) + self.assertEqual(count + 1, RestService.objects.all().count()) meta = MetaData.objects.filter(object_id=self.xform.id, data_type='textit') - self.assertEquals(len(meta), 1) + self.assertEqual(len(meta), 1) rs = RestService.objects.last() expected_dict = { @@ -124,8 +124,8 @@ def test_delete_textit_service(self): request = self.factory.delete('/', **self.extra) response = self.view(request, pk=rest['id']) - self.assertEquals(response.status_code, 204) - self.assertEquals(count - 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 204) + self.assertEqual(count - 1, RestService.objects.all().count()) a_meta_count = MetaData.objects.filter(object_id=self.xform.id, data_type='textit').count() self.assertEqual(meta_count - 1, a_meta_count) @@ -149,8 +149,8 @@ def test_update(self): request = self.factory.put('/', data=post_data, **self.extra) response = self.view(request, pk=rest.pk) - self.assertEquals(response.status_code, 200) - self.assertEquals(response.data['name'], "textit") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['name'], "textit") self.assertEqual(response.data['flow_title'], 'test-flow') metadata_count = MetaData.objects.count() @@ -180,8 +180,8 @@ def test_update_with_errors(self): request = self.factory.get('/', **self.extra) response = self.view(request, pk=rest.get('id')) - self.assertEquals(response.status_code, 400) - self.assertEquals(response.data, + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, [u"Error occurred when loading textit service." u"Resolve by updating auth_token, flow_uuid and " u"contacts fields"]) @@ -198,7 +198,7 @@ def test_update_with_errors(self): request = self.factory.put('/', data=post_data, **self.extra) response = self.view(request, pk=rest.get('id')) - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) def test_delete(self): rest = RestService(name="testservice", @@ -211,8 +211,8 @@ def test_delete(self): request = self.factory.delete('/', **self.extra) response = self.view(request, pk=rest.pk) - self.assertEquals(response.status_code, 204) - self.assertEquals(count - 1, RestService.objects.all().count()) + self.assertEqual(response.status_code, 204) + self.assertEqual(count - 1, RestService.objects.all().count()) def test_retrieve(self): rest = RestService(name="testservice", @@ -233,9 +233,9 @@ def test_retrieve(self): } response.data.pop('date_modified') response.data.pop('date_created') - self.assertEquals(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.assertEquals(response.data, data) + self.assertEqual(response.data, data) def test_duplicate_rest_service(self): post_data = { @@ -249,7 +249,7 @@ def test_duplicate_rest_service(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 201) + self.assertEqual(response.status_code, 201) post_data = { "name": "textit", @@ -262,7 +262,7 @@ def test_duplicate_rest_service(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch('requests.post') @@ -281,7 +281,7 @@ def test_textit_flow(self, mock_http): self._make_submissions() self.assertTrue(mock_http.called) - self.assertEquals(mock_http.call_count, 4) + self.assertEqual(mock_http.call_count, 4) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch('requests.post') @@ -313,6 +313,6 @@ def test_create_rest_service_invalid_form_id(self): request = self.factory.post('/', data=post_data, **self.extra) response = self.view(request) - self.assertEquals(response.status_code, 400) + self.assertEqual(response.status_code, 400) self.assertEqual(response.data, {'xform': [u'Invalid form id']}) - self.assertEquals(count, RestService.objects.all().count()) + self.assertEqual(count, RestService.objects.all().count()) diff --git a/onadata/apps/viewer/tests/test_export_list.py b/onadata/apps/viewer/tests/test_export_list.py index 468a75ecac..925c22276c 100644 --- a/onadata/apps/viewer/tests/test_export_list.py +++ b/onadata/apps/viewer/tests/test_export_list.py @@ -132,7 +132,7 @@ def test_external_export_list(self): response = self.client.get(url, custom_params) self.assertEqual(response.status_code, 200) count1 = len(Export.objects.all()) - self.assertEquals(count + 1, count1) + self.assertEqual(count + 1, count1) def test_external_export_list_no_template(self): kwargs = {'username': self.user.username, @@ -143,10 +143,10 @@ def test_external_export_list_no_template(self): count = len(Export.objects.all()) response = self.client.get(url) self.assertEqual(response.status_code, 403) - self.assertEquals(response.content.decode('utf-8'), + self.assertEqual(response.content.decode('utf-8'), u'No XLS Template set.') count1 = len(Export.objects.all()) - self.assertEquals(count, count1) + self.assertEqual(count, count1) class TestDataExportURL(TestBase): diff --git a/onadata/apps/viewer/tests/test_exports.py b/onadata/apps/viewer/tests/test_exports.py index d9a5ece118..4d3846bfcf 100644 --- a/onadata/apps/viewer/tests/test_exports.py +++ b/onadata/apps/viewer/tests/test_exports.py @@ -1336,7 +1336,7 @@ def test_create_external_export_without_template(self): response = self.client.post(create_export_url) self.assertEqual(response.status_code, 403) - self.assertEquals(response.content, b'No XLS Template set.') + self.assertEqual(response.content, b'No XLS Template set.') self.assertEqual(Export.objects.count(), num_exports) def test_all_keys_cleaned_of_slashes(self): @@ -1466,4 +1466,4 @@ def test_all_keys_cleaned_of_slashes(self): } result_data = clean_keys_of_slashes(data) - self.assertEquals(expected_data, result_data) + self.assertEqual(expected_data, result_data) diff --git a/onadata/apps/viewer/tests/test_tasks.py b/onadata/apps/viewer/tests/test_tasks.py index a080de361e..dec4631144 100644 --- a/onadata/apps/viewer/tests/test_tasks.py +++ b/onadata/apps/viewer/tests/test_tasks.py @@ -42,7 +42,7 @@ def test_create_async(self): export = result[0] self.assertTrue(export.id) self.assertIn("username", options) - self.assertEquals(options.get("id_string"), self.xform.id_string) + self.assertEqual(options.get("id_string"), self.xform.id_string) def test_mark_expired_pending_exports_as_failed(self): self._publish_transportation_form_and_submit_instance() @@ -56,7 +56,7 @@ def test_mark_expired_pending_exports_as_failed(self): export.save() mark_expired_pending_exports_as_failed() export = Export.objects.filter(pk=export.pk).first() - self.assertEquals(export.internal_status, Export.FAILED) + self.assertEqual(export.internal_status, Export.FAILED) def test_delete_expired_failed_exports(self): self._publish_transportation_form_and_submit_instance() @@ -70,4 +70,4 @@ def test_delete_expired_failed_exports(self): export.save() pk = export.pk delete_expired_failed_exports() - self.assertEquals(Export.objects.filter(pk=pk).first(), None) + self.assertEqual(Export.objects.filter(pk=pk).first(), None) diff --git a/onadata/libs/tests/filters/test_instance_permission_filter.py b/onadata/libs/tests/filters/test_instance_permission_filter.py index 81203bae54..79ddbcf541 100644 --- a/onadata/libs/tests/filters/test_instance_permission_filter.py +++ b/onadata/libs/tests/filters/test_instance_permission_filter.py @@ -79,7 +79,7 @@ def test_metadata_filter_for_user_with_xform_perms(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 1) + self.assertEqual(len(response.data), 1) def test_metadata_filter_for_user_without_xform_perms(self): self._create_user_and_login("alice", "password") @@ -91,7 +91,7 @@ def test_metadata_filter_for_user_without_xform_perms(self): request = self.factory.get('/', data=params, **extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_for_foreign_instance_request(self): self._create_user_and_login("alice", "password") @@ -110,7 +110,7 @@ def test_filter_for_foreign_instance_request(self): request = self.factory.get('/', data=params, **extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_for_dataview_metadata_instance_request(self): # """Dataview IDs should not yield any submission metadata""" @@ -128,7 +128,7 @@ def test_filter_for_dataview_metadata_instance_request(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_given_user_without_permissions_to_xform(self): """Instance metadata isn't listed for users without form perms""" @@ -152,7 +152,7 @@ def test_filter_given_user_without_permissions_to_xform(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) def test_filter_given_dataview_in_project_without_instance(self): """Meta data for submissions shouldn't be accessible from dataview""" @@ -173,4 +173,4 @@ def test_filter_given_dataview_in_project_without_instance(self): request = self.factory.get('/', data=params, **self.extra) response = self.view(request) - self.assertEquals(len(response.data), 0) + self.assertEqual(len(response.data), 0) diff --git a/onadata/libs/tests/serializers/test_attachment_serializer.py b/onadata/libs/tests/serializers/test_attachment_serializer.py index ead4e8adb3..3c572bc009 100644 --- a/onadata/libs/tests/serializers/test_attachment_serializer.py +++ b/onadata/libs/tests/serializers/test_attachment_serializer.py @@ -59,9 +59,9 @@ def setUp(self): def test_get_field_xpath_of_an_object(self): path = get_path(self.data, self.question, []) - self.assertEquals(path, "group1/group2/photograph") + self.assertEqual(path, "group1/group2/photograph") # call 'get_path' twice to ensure the incorrect path bug has been # sorted; calling it the second time initially would return the # following path: "group1/group2/photograph/group1/group2/photograph" path = get_path(self.data, self.question, []) - self.assertEquals(path, "group1/group2/photograph") + self.assertEqual(path, "group1/group2/photograph") diff --git a/onadata/libs/tests/serializers/test_dataview_serializer.py b/onadata/libs/tests/serializers/test_dataview_serializer.py index af69656cbc..4f582ef6c0 100644 --- a/onadata/libs/tests/serializers/test_dataview_serializer.py +++ b/onadata/libs/tests/serializers/test_dataview_serializer.py @@ -73,7 +73,7 @@ def test_name_and_xform_are_unique(self): self.assertTrue(is_valid) serializer.save() - self.assertEquals(DataView.objects.count(), 1) + self.assertEqual(DataView.objects.count(), 1) # Try and save the same data again and confirm it fails serializer = DataViewSerializer( @@ -86,4 +86,4 @@ def test_name_and_xform_are_unique(self): serializer.errors.get( 'non_field_errors')[0].title() == expected_error_msg - self.assertEquals(DataView.objects.count(), 1) + self.assertEqual(DataView.objects.count(), 1) diff --git a/onadata/libs/tests/serializers/test_metadata_serializer.py b/onadata/libs/tests/serializers/test_metadata_serializer.py index 0190eacbea..54f0d7e3fb 100644 --- a/onadata/libs/tests/serializers/test_metadata_serializer.py +++ b/onadata/libs/tests/serializers/test_metadata_serializer.py @@ -88,5 +88,5 @@ def test_svg_media_files(self): } serializer = MetaDataSerializer(data=data) self.assertTrue(serializer.is_valid()) - self.assertEquals(serializer.validated_data['data_file_type'], + self.assertEqual(serializer.validated_data['data_file_type'], 'image/svg+xml') diff --git a/onadata/libs/tests/test_authentication.py b/onadata/libs/tests/test_authentication.py index 2dc55a94a8..c7c1e4e956 100644 --- a/onadata/libs/tests/test_authentication.py +++ b/onadata/libs/tests/test_authentication.py @@ -103,8 +103,8 @@ def test_authenticates_if_token_is_valid(self): returned_user, returned_token, ) = self.temp_token_authentication.authenticate_credentials(token.key) - self.assertEquals(user, returned_user) - self.assertEquals(token, returned_token) + self.assertEqual(user, returned_user) + self.assertEqual(token, returned_token) class TestTempTokenURLParameterAuthentication(TestCase): diff --git a/onadata/libs/tests/test_renderers.py b/onadata/libs/tests/test_renderers.py index bc777a56bd..037c6d6aba 100644 --- a/onadata/libs/tests/test_renderers.py +++ b/onadata/libs/tests/test_renderers.py @@ -30,7 +30,7 @@ def test_floip_rows_list(self): ['2018-03-05T13:57:26+00:00', 19, None, 1, 'age', 10, None] ] # yapf: disable result = [_ for _ in floip_rows_list(data)] - self.assertEquals(result, expected_data) + self.assertEqual(result, expected_data) def test_floip_rows_list_w_flow_fields(self): # pylint: disable=C0103 """ @@ -50,4 +50,4 @@ def test_floip_rows_list_w_flow_fields(self): # pylint: disable=C0103 ['2018-03-05T13:57:26+00:00', 34, '789', '345', 'age', 10, None] ] # yapf: disable result = [_ for _ in floip_rows_list(data)] - self.assertEquals(result, expected_data) + self.assertEqual(result, expected_data) diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index 9f1bb9736c..2fd263e523 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -76,7 +76,7 @@ def test_process_async_export_returns_existing_export(self): resp = process_async_export( request, self.xform, export_type, options=options) - self.assertEquals(resp['job_status'], status_msg[SUCCESSFUL]) + self.assertEqual(resp['job_status'], status_msg[SUCCESSFUL]) self.assertIn("export_url", resp) # pylint: disable=invalid-name diff --git a/onadata/libs/tests/utils/test_chart_tools.py b/onadata/libs/tests/utils/test_chart_tools.py index c305cacf8f..dd76714d76 100644 --- a/onadata/libs/tests/utils/test_chart_tools.py +++ b/onadata/libs/tests/utils/test_chart_tools.py @@ -85,7 +85,7 @@ def test_build_chart_data_for_fields_with_accents(self): count = XForm.objects.count() self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), count + 1) + self.assertEqual(XForm.objects.count(), count + 1) xform = XForm.objects.get(id_string='sample_accent') self.assertEqual(xform.title, "sample_accent") @@ -113,7 +113,7 @@ def test_build_chart_data_for_fields_with_apostrophies(self): count = XForm.objects.count() self._publish_xls_file(xls_path) - self.assertEquals(XForm.objects.count(), count + 1) + self.assertEqual(XForm.objects.count(), count + 1) xform = XForm.objects.get(id_string='sample_accent') self.assertEqual(xform.title, "sample_accent") diff --git a/onadata/libs/tests/utils/test_csv_builder.py b/onadata/libs/tests/utils/test_csv_builder.py index e87ffcd233..095c7bdcb3 100644 --- a/onadata/libs/tests/utils/test_csv_builder.py +++ b/onadata/libs/tests/utils/test_csv_builder.py @@ -1220,7 +1220,7 @@ def test_export_data_for_xforms_without_submissions(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder = CSVDataFrameBuilder( self.user.username, self.xform.id_string, include_images=False) @@ -1253,7 +1253,7 @@ def test_export_data_for_xforms_with_newer_submissions(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder = CSVDataFrameBuilder( self.user.username, self.xform.id_string, include_images=False) @@ -1320,7 +1320,7 @@ def test_export_raises_NoRecordsFound_for_form_without_instances(self): self._publish_xls_fixture_set_xform(fixture) # Confirm form has not submissions so far - self.assertEquals(self.xform.instances.count(), 0) + self.assertEqual(self.xform.instances.count(), 0) # Generate csv export for form csv_df_builder_1 = CSVDataFrameBuilder( self.user.username, diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index ab81c7f636..90b90d4a03 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -31,7 +31,7 @@ def test_generate_content_disposition_header(self): "%s-%s.%s" % (file_name, date_pattern, extension) return_value_with_no_name = \ generate_content_disposition_header(None, extension) - self.assertEquals(return_value_with_no_name, "attachment;") + self.assertEqual(return_value_with_no_name, "attachment;") return_value_with_name_and_no_show_date = \ generate_content_disposition_header(file_name, extension) @@ -79,11 +79,11 @@ def test_attachment_tracking(self): BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) @@ -95,11 +95,11 @@ def test_attachment_tracking(self): media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received) media2_file = django_file( path=file2_path, field_name="image2", content_type="image/jpeg") @@ -109,11 +109,11 @@ def test_attachment_tracking(self): media_files=[media2_file]) instance3 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance3.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance3.json[TOTAL_MEDIA], 2) - self.assertEquals(instance3.json[MEDIA_COUNT], 2) - self.assertEquals(instance3.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance3.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance3.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance3.json[TOTAL_MEDIA], 2) + self.assertEqual(instance3.json[MEDIA_COUNT], 2) + self.assertEqual(instance3.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance3.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual(instance3.json[MEDIA_ALL_RECEIVED], instance3.media_all_received) def test_attachment_tracking_for_repeats(self): @@ -155,11 +155,11 @@ def test_attachment_tracking_for_repeats(self): BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) @@ -171,11 +171,11 @@ def test_attachment_tracking_for_repeats(self): media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received) def test_attachment_tracking_for_nested_repeats(self): @@ -219,11 +219,11 @@ def test_attachment_tracking_for_nested_repeats(self): BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) @@ -235,11 +235,11 @@ def test_attachment_tracking_for_nested_repeats(self): media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 2) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 2) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received) def test_replaced_attachments_not_tracked(self): @@ -283,11 +283,11 @@ def test_replaced_attachments_not_tracked(self): self.assertEqual( instance.attachments.filter(deleted_at__isnull=True).count(), 2) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 2) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 2) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) patch_value = 'onadata.apps.logger.models.Instance.get_expected_media' with patch(patch_value) as get_expected_media: @@ -318,13 +318,13 @@ def test_replaced_attachments_not_tracked(self): self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) # Test that only one attachment is recognised for this submission # Since the file is no longer present in the submission - self.assertEquals(instance2.json[TOTAL_MEDIA], 1) - self.assertEquals(instance2.json[MEDIA_COUNT], 1) - self.assertEquals( + self.assertEqual(instance2.json[TOTAL_MEDIA], 1) + self.assertEqual(instance2.json[MEDIA_COUNT], 1) + self.assertEqual( instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals( + self.assertEqual( instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals( + self.assertEqual( instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received) @@ -361,11 +361,11 @@ def test_attachment_tracking_duplicate(self): BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) media2_file = django_file( path=file_path, field_name="image1", content_type="image/jpeg") @@ -375,11 +375,11 @@ def test_attachment_tracking_duplicate(self): media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertFalse(instance2.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance2.json[TOTAL_MEDIA], 2) - self.assertEquals(instance2.json[MEDIA_COUNT], 1) - self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance2.json[TOTAL_MEDIA], 2) + self.assertEqual(instance2.json[MEDIA_COUNT], 1) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) + self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received) def test_attachment_tracking_not_in_submission(self): @@ -418,11 +418,11 @@ def test_attachment_tracking_not_in_submission(self): BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file, media2_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 2) - self.assertEquals(instance.json[MEDIA_COUNT], 1) - self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) - self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], + self.assertEqual(instance.json[TOTAL_MEDIA], 2) + self.assertEqual(instance.json[MEDIA_COUNT], 1) + self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) + self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) def test_get_first_record(self): diff --git a/onadata/libs/tests/utils/test_model_tools.py b/onadata/libs/tests/utils/test_model_tools.py index ec187dd476..1848db1d91 100644 --- a/onadata/libs/tests/utils/test_model_tools.py +++ b/onadata/libs/tests/utils/test_model_tools.py @@ -14,7 +14,7 @@ def test_queryset_iterator(self): username='test_2', password='test_2', email='test_2@test.com') user_model.objects.create_user( username='test_3', password='test_3', email='test@test_3.com') - self.assertEquals( + self.assertEqual( 'generator', queryset_iterator( user_model.objects.all(), chunksize=1).__class__.__name__ From c558cedf2dd917ddfda24c1d8a1e67ec52567a0b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 10:46:43 +0300 Subject: [PATCH 091/234] batch: cleanup --- .../tests/viewsets/test_abstract_viewset.py | 4 +- onadata/apps/main/tests/test_base.py | 92 ++++++++++++------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index f3a3180e02..93f1e6a1f2 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -189,8 +189,8 @@ def _org_create(self, org_data=None): ) response = view(request) self.assertEqual(response.status_code, 201) - data["url"] = "http://testserver/api/v1/orgs/{data['org']}" - data["user"] = "http://testserver/api/v1/users/{data['org']}" + data["url"] = f"http://testserver/api/v1/orgs/{data['org']}" + data["user"] = f"http://testserver/api/v1/users/{data['org']}" data["creator"] = "http://testserver/api/v1/users/bob" self.assertDictContainsSubset(data, response.data) # pylint: disable=attribute-defined-outside-init diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index 332964273b..9431dd9a60 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +TestBase - a TestCase base class. +""" from __future__ import unicode_literals import base64 @@ -5,15 +9,11 @@ import os import re import socket -from builtins import open -from six.moves.urllib.error import URLError -from six.moves.urllib.request import urlopen from io import StringIO from tempfile import NamedTemporaryFile from django.conf import settings -from django.contrib.auth import authenticate -from django.contrib.auth.models import User +from django.contrib.auth import authenticate, get_user_model from django.core.files.uploadedfile import InMemoryUploadedFile from django.test import RequestFactory, TransactionTestCase from django.test.client import Client @@ -21,22 +21,32 @@ from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth -from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from rest_framework.test import APIRequestFactory +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.logger.views import submission from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models import DataDictionary +from onadata.libs.test_utils.pyxform_test_case import PyxformMarkdown from onadata.libs.utils.common_tools import ( filename_from_disposition, get_response_content, ) from onadata.libs.utils.user_auth import get_user_default_project +# pylint: disable=invalid-name +User = get_user_model() + +# pylint: disable=too-many-instance-attributes class TestBase(PyxformMarkdown, TransactionTestCase): + """ + A TransactionTestCase base class for test modules. + """ + maxDiff = None surveys = [ "transport_2011-07-25_19-05-49", @@ -52,11 +62,13 @@ def setUp(self): self.base_url = "http://testserver" self.factory = RequestFactory() + # pylint: disable=no-self-use def _fixture_path(self, *args): return os.path.join(os.path.dirname(__file__), "fixtures", *args) + # pylint: disable=no-self-use def _create_user(self, username, password, create_profile=False): - user, created = User.objects.get_or_create(username=username) + user, _created = User.objects.get_or_create(username=username) user.set_password(password) user.save() @@ -68,6 +80,7 @@ def _create_user(self, username, password, create_profile=False): return user + # pylint: disable=no-self-use def _login(self, username, password): client = Client() assert client.login(username=username, password=password) @@ -88,7 +101,7 @@ def _create_user_and_login(self, username="bob", password="bob", factory=None): self.anon = Client() def _publish_xls_file(self, path): - if not path.startswith("/%s/" % self.user.username): + if not path.startswith(f"/{self.user.username}/"): path = os.path.join(self.this_directory, path) with open(path, "rb") as f: xls_file = InMemoryUploadedFile( @@ -100,6 +113,7 @@ def _publish_xls_file(self, path): None, ) if not hasattr(self, "project"): + # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) DataDictionary.objects.create( @@ -126,6 +140,7 @@ def _publish_xls_file_and_set_xform(self, path): count = XForm.objects.count() self._publish_xls_file(path) self.assertEqual(XForm.objects.count(), count + 1) + # pylint: disable=attribute-defined-outside-init self.xform = XForm.objects.order_by("pk").reverse()[0] def _share_form_data(self, id_string="transportation_2011_07_25"): @@ -140,6 +155,7 @@ def _publish_transportation_form(self): count = XForm.objects.count() TestBase._publish_xls_file(self, xls_path) self.assertEqual(XForm.objects.count(), count + 1) + # pylint: disable=attribute-defined-outside-init self.xform = XForm.objects.order_by("pk").reverse()[0] def _submit_transport_instance(self, survey_at=0): @@ -188,6 +204,7 @@ def _submit_transport_instance_w_attachment(self, survey_at=0): media_file, ), ) + # pylint: disable=attribute-defined-outside-init self.attachment = Attachment.objects.all().reverse()[0] self.attachment_media_file = self.attachment.media_file @@ -204,6 +221,7 @@ def _make_submissions_gps(self): path = self._fixture_path("gps", "instances", survey + ".xml") self._make_submission(path) + # pylint: disable=too-many-arguments,too-many-locals,unused-argument def _make_submission( self, path, @@ -222,16 +240,15 @@ def _make_submission( tmp_file = None if add_uuid: - tmp_file = NamedTemporaryFile(delete=False, mode="w") - split_xml = None + with NamedTemporaryFile(delete=False, mode="w") as tmp_file: + split_xml = None - with open(path, encoding="utf-8") as _file: - split_xml = re.split(r"()", _file.read()) + with open(path, encoding="utf-8") as _file: + split_xml = re.split(r"()", _file.read()) - split_xml[1:1] = ["%s" % self.xform.uuid] - tmp_file.write("".join(split_xml)) - path = tmp_file.name - tmp_file.close() + split_xml[1:1] = [f"{self.xform.uuid}"] + tmp_file.write("".join(split_xml)) + path = tmp_file.name with open(path, encoding="utf-8") as f: post_data = {"xml_submission_file": f} @@ -239,12 +256,13 @@ def _make_submission( if username is None: username = self.user.username - url_prefix = "%s/" % username if username else "" - url = "/%ssubmission" % url_prefix + url_prefix = f"{username if username else ''}/" + url = f"/{url_prefix}submission" request = self.factory.post(url, post_data) request.user = authenticate(username=auth.username, password=auth.password) + # pylint: disable=attribute-defined-outside-init self.response = submission(request, username=username) if auth and self.response.status_code == 401: @@ -267,16 +285,19 @@ def _make_submission_w_attachment(self, path, attachment_path): data = {"xml_submission_file": f} if attachment_path is not None: if isinstance(attachment_path, list): - for c in range(len(attachment_path)): - data["media_file_{}".format(c)] = open(attachment_path[c], "rb") + for index, item_path in enumerate(attachment_path): + # pylint: disable=consider-using-with + data[f"media_file_{index}"] = open(item_path, "rb") else: + # pylint: disable=consider-using-with data["media_file"] = open(attachment_path, "rb") - url = "/%s/submission" % self.user.username + url = f"/{self.user.username}/submission" auth = DigestAuth("bob", "bob") self.factory = APIRequestFactory() request = self.factory.post(url, data) request.user = authenticate(username="bob", password="bob") + # pylint: disable=attribute-defined-outside-init self.response = submission(request, username=self.user.username) if auth and self.response.status_code == 401: @@ -316,8 +337,8 @@ def _make_submissions(self, username=None, add_uuid=False, should_store=True): def _check_url(self, url, timeout=1): try: - urlopen(url, timeout=timeout) - return True + with urlopen(url, timeout=timeout): + return True except (URLError, socket.timeout): pass return False @@ -329,13 +350,16 @@ def _internet_on(self, url="http://74.125.113.99"): def _set_auth_headers(self, username, password): return { "HTTP_AUTHORIZATION": "Basic " - + base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode( + + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode( "utf-8" ), } - def _get_authenticated_client(self, url, username="bob", password="bob", extra={}): + def _get_authenticated_client( + self, url, username="bob", password="bob", extra=None + ): client = DigestClient() + extra = {} if extra is None else extra # request with no credentials req = client.get(url, {}, **extra) self.assertEqual(req.status_code, 401) @@ -348,7 +372,7 @@ def _set_mock_time(self, mock_time): mock_time.return_value = current_time def _set_require_auth(self, auth=True): - profile, created = UserProfile.objects.get_or_create(user=self.user) + profile, _created = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = auth profile.save() @@ -372,7 +396,7 @@ def _publish_submit_geojson(self): self._publish_xls_file_and_set_xform(path) view = XFormViewSet.as_view({"post": "csv_import"}) - csv_import = open( + with open( os.path.join( settings.PROJECT_ROOT, "apps", @@ -383,11 +407,11 @@ def _publish_submit_geojson(self): "GeoLocationForm_2015_01_15_01_28_45.csv", ), encoding="utf-8", - ) - post_data = {"csv_file": csv_import} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request, pk=self.xform.id) - self.assertEqual(response.status_code, 200) + ) as csv_import: + post_data = {"csv_file": csv_import} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request, pk=self.xform.id) + self.assertEqual(response.status_code, 200) def _publish_markdown(self, md_xlsform, user, project=None, **kwargs): """ @@ -419,7 +443,7 @@ def _test_csv_response(self, response, csv_file_path): data = get_response_content(response) reader = csv.DictReader(StringIO(data)) - data = [_ for _ in reader] + data = list(reader) with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): @@ -429,7 +453,7 @@ def _test_csv_response(self, response, csv_file_path): def _test_csv_files(self, csv_file, csv_file_path): reader = csv.DictReader(csv_file) - data = [_ for _ in reader] + data = list(reader) with open(csv_file_path, encoding="utf-8") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): From 039ab1a76696af6f07a3dab99c8eaf6997c67e7b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 11:26:39 +0300 Subject: [PATCH 092/234] batch: cleanup --- .../api/tests/viewsets/test_data_viewset.py | 2244 +++++++++-------- .../test_organization_profile_viewset.py | 1142 ++++----- .../api/tests/viewsets/test_widget_viewset.py | 734 +++--- .../apps/logger/tests/test_form_submission.py | 315 +-- onadata/apps/main/tests/test_form_metadata.py | 379 +-- onadata/apps/main/tests/test_metadata.py | 135 +- .../restservice/tests/test_restservice.py | 112 +- .../viewsets/test_restservicesviewset.py | 245 +- onadata/apps/viewer/tests/test_export_list.py | 250 +- .../serializers/test_metadata_serializer.py | 67 +- onadata/libs/tests/utils/test_logger_tools.py | 389 +-- onadata/libs/utils/viewer_tools.py | 102 +- 12 files changed, 3207 insertions(+), 2907 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index a0019cf8f5..ff2140f9a7 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test /data API endpoint implementation. +""" from __future__ import unicode_literals import datetime @@ -21,15 +25,18 @@ from httmock import urlmatch, HTTMock from mock import patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - enketo_urls_mock +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import enketo_urls_mock from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.logger.models import \ - Instance, SurveyType, XForm, Attachment, SubmissionReview +from onadata.apps.logger.models import ( + Instance, + SurveyType, + XForm, + Attachment, + SubmissionReview, +) from onadata.apps.logger.models.instance import InstanceHistory from onadata.apps.logger.models.instance import get_attachment_url from onadata.apps.main import tests as main_tests @@ -38,74 +45,85 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED from onadata.libs import permissions as role -from onadata.libs.permissions import ReadOnlyRole, EditorRole, \ - EditorMinorRole, DataEntryOnlyRole, DataEntryMinorRole, \ - ManagerRole -from onadata.libs.serializers.submission_review_serializer import \ - SubmissionReviewSerializer +from onadata.libs.permissions import ( + ReadOnlyRole, + EditorRole, + EditorMinorRole, + DataEntryOnlyRole, + DataEntryMinorRole, + ManagerRole, +) +from onadata.libs.serializers.submission_review_serializer import ( + SubmissionReviewSerializer, +) from onadata.libs.utils.common_tags import MONGO_STRFTIME from onadata.libs.utils.logger_tools import create_instance -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_edit_mock(url, request): response = requests.Response() response.status_code = 201 response._content = ( '{"edit_url": "https://hmh2a.enketo.ona.io/edit/XA0bG8D' - 'f?instance_id=672927e3-9ad4-42bb-9538-388ea1fb6699&retu' - 'rnUrl=http://test.io/test_url", "code": 201}') + "f?instance_id=672927e3-9ad4-42bb-9538-388ea1fb6699&retu" + 'rnUrl=http://test.io/test_url", "code": 201}' + ) return response -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_mock_http_413(url, request): response = requests.Response() response.status_code = 413 - response._content = '' + response._content = "" return response def _data_list(formid): - return [{ - 'id': formid, - 'id_string': 'transportation_2011_07_25', - 'title': 'transportation_2011_07_25', - 'description': '', - 'url': 'http://testserver/api/v1/data/%s' % formid - }] + return [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "title": "transportation_2011_07_25", + "description": "", + "url": "http://testserver/api/v1/data/%s" % formid, + } + ] def _data_instance(dataid): return { - '_bamboo_dataset_id': '', - '_attachments': [], - '_geolocation': [None, None], - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_status': 'submitted_via_web', - '_id': dataid + "_bamboo_dataset_id": "", + "_attachments": [], + "_geolocation": [None, None], + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_status": "submitted_via_web", + "_id": dataid, } +# pylint: disable=too-many-public-methods class TestDataViewSet(TestBase): + """ + Test /data API endpoint implementation. + """ def setUp(self): super(self.__class__, self).setUp() self._create_user_and_login() self._publish_transportation_form() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} def test_data(self): """Test DataViewSet list""" self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk data = _data_list(formid) @@ -115,31 +133,31 @@ def test_data(self): self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) @override_settings(STREAM_DATA=True) def test_data_streaming(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk data = _data_list(formid) @@ -149,122 +167,124 @@ def test_data_streaming(self): response = view(request, pk=formid) self.assertEqual(response.status_code, 200) streaming_data = json.loads( - ''.join([i.decode('utf-8') for i in response.streaming_content]) + "".join([i.decode("utf-8") for i in response.streaming_content]) ) self.assertIsInstance(streaming_data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(streaming_data, key=lambda x: x['_id'])[0]) + data, sorted(streaming_data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) def test_catch_data_error(self): - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - query_str = [('\'{"_submission_time":{' - '"$and":[{"$gte":"2015-11-15T00:00:00"},' - '{"$lt":"2015-11-16T00:00:00"}]}}')] + query_str = [ + ( + '\'{"_submission_time":{' + '"$and":[{"$gte":"2015-11-15T00:00:00"},' + '{"$lt":"2015-11-16T00:00:00"}]}}' + ) + ] data = { - 'query': query_str, + "query": query_str, } - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data.get('detail'), - 'invalid regular expression: invalid character range\n') + response.data.get("detail"), + "invalid regular expression: invalid character range\n", + ) def test_data_list_with_xform_in_delete_async_queue(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(len(response.data), initial_count - 1) def test_numeric_types_are_rendered_as_required(self): tutorial_folder = os.path.join( - os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'tutorial') - self._publish_xls_file_and_set_xform(os.path.join(tutorial_folder, - 'tutorial.xlsx')) + os.path.dirname(__file__), "..", "fixtures", "forms", "tutorial" + ) + self._publish_xls_file_and_set_xform( + os.path.join(tutorial_folder, "tutorial.xlsx") + ) - instance_path = os.path.join(tutorial_folder, 'instances', '1.xml') - create_instance(self.user.username, open(instance_path, 'rb'), []) + instance_path = os.path.join(tutorial_folder, "instances", "1.xml") + create_instance(self.user.username, open(instance_path, "rb"), []) self.assertEqual(self.xform.instances.count(), 1) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, 200) # check that ONLY values with numeric and decimal types are converted - self.assertEqual(response.data[0].get('age'), 35) - self.assertEqual(response.data[0].get('net_worth'), 100000.00) - self.assertEqual(response.data[0].get('imei'), '351746052009472') + self.assertEqual(response.data[0].get("age"), 35) + self.assertEqual(response.data[0].get("net_worth"), 100000.00) + self.assertEqual(response.data[0].get("imei"), "351746052009472") def test_data_jsonp(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) - response = view(request, pk=formid, format='jsonp') + request = self.factory.get("/", **self.extra) + response = view(request, pk=formid, format="jsonp") self.assertEqual(response.status_code, 200) response.render() - content = response.content.decode('utf-8') - self.assertTrue(content.startswith('callback(')) - self.assertTrue(content.endswith(');')) + content = response.content.decode("utf-8") + self.assertTrue(content.startswith("callback(")) + self.assertTrue(content.endswith(");")) self.assertEqual(len(response.data), 4) def _assign_user_role(self, user, role): # share bob's project with alice and give alice an editor role - data = {'username': user.username, 'role': role.name} - request = self.factory.put('/', data=data, **self.extra) - xform_view = XFormViewSet.as_view({ - 'put': 'share' - }) + data = {"username": user.username, "role": role.name} + request = self.factory.put("/", data=data, **self.extra) + xform_view = XFormViewSet.as_view({"put": "share"}) response = xform_view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 204) - self.assertTrue( - role.user_has_role(user, self.xform) - ) + self.assertTrue(role.user_has_role(user, self.xform)) def test_returned_data_is_based_on_form_permissions(self): # create a form and make submissions to it self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False @@ -276,23 +296,21 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, DataEntryOnlyRole) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) data = {"start": 1, "limit": 4} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) data = {"sort": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) @@ -300,7 +318,7 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, EditorMinorRole) # check that by default, alice can be able to access all the data - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) @@ -317,33 +335,39 @@ def test_returned_data_is_based_on_form_permissions(self): # check that 2 instances were 'submitted by' alice instances_submitted_by_alice = self.xform.instances.filter( - user=user_alice).count() + user=user_alice + ).count() self.assertTrue(instances_submitted_by_alice, 2) # check that alice will only be able to see the data she submitted - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # check that alice views only her data when querying data - data = {"start": 0, "limit": 100, "sort": '{"_submission_time":1}', - "query": "c"} - request = self.factory.get('/', data=data, **alices_extra) + data = { + "start": 0, + "limit": 100, + "sort": '{"_submission_time":1}', + "query": "c", + } + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) self.assertListEqual( - [r['_submitted_by'] for r in response.data], ['alice', 'alice']) + [r["_submitted_by"] for r in response.data], ["alice", "alice"] + ) data = {"start": 1, "limit": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) data = {"sort": 1} - request = self.factory.get('/', data=data, **alices_extra) + request = self.factory.get("/", data=data, **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) @@ -354,14 +378,14 @@ def test_returned_data_is_based_on_form_permissions(self): self._assign_user_role(user_alice, EditorRole) - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) self._assign_user_role(user_alice, ReadOnlyRole) - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -369,55 +393,44 @@ def test_returned_data_is_based_on_form_permissions(self): def test_xform_meta_permissions_not_affected_w_projects_perms(self): # create a form and make submissions to it self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False profile.save() - data = {'username': user_alice.username, 'role': EditorRole.name} - request = self.factory.put('/', data=data, **self.extra) - project_view = ProjectViewSet.as_view({ - 'put': 'share' - }) + data = {"username": user_alice.username, "role": EditorRole.name} + request = self.factory.put("/", data=data, **self.extra) + project_view = ProjectViewSet.as_view({"put": "share"}) response = project_view(request, pk=self.project.pk) self.assertEqual(response.status_code, 204) - self.assertTrue( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertTrue(EditorRole.user_has_role(user_alice, self.xform)) self._assign_user_role(user_alice, EditorMinorRole) - MetaData.xform_meta_permission(self.xform, - data_value='editor-minor|dataentry') + MetaData.xform_meta_permission(self.xform, data_value="editor-minor|dataentry") - self.assertFalse( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertFalse(EditorRole.user_has_role(user_alice, self.xform)) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertFalse( - EditorRole.user_has_role(user_alice, self.xform) - ) + self.assertFalse(EditorRole.user_has_role(user_alice, self.xform)) self.assertEqual(len(response.data), 0) def test_data_entryonly_can_submit_but_not_view(self): # create user alice - user_alice = self._create_user('alice', 'alice') + user_alice = self._create_user("alice", "alice") # create user profile and set require_auth to false for tests profile, created = UserProfile.objects.get_or_create(user=user_alice) profile.require_auth = False @@ -430,229 +443,233 @@ def test_data_entryonly_can_submit_but_not_view(self): DataEntryOnlyRole.add(user_alice, self.xform) DataEntryOnlyRole.add(user_alice, self.project) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") - paths = [os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', s, s + '.xml') for s in self.surveys] + paths = [ + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + for s in self.surveys + ] for path in paths: self._make_submission(path, auth=auth) - alices_extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % user_alice.auth_token.key - } + alices_extra = {"HTTP_AUTHORIZATION": "Token %s" % user_alice.auth_token.key} - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 404) DataEntryMinorRole.add(user_alice, self.xform) DataEntryMinorRole.add(user_alice, self.project) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - request = self.factory.get('/', **alices_extra) + request = self.factory.get("/", **alices_extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_pagination(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no page param no pagination - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - request = self.factory.get('/', data={"page": "1", "page_size": 2}, - **self.extra) + request = self.factory.get( + "/", data={"page": "1", "page_size": 2}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # Link pagination headers are present and work as intended - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - '; rel="next"' + response["Link"], '; rel="next"' ) - request = self.factory.get( - '/', data={"page": 2, "page_size": 1}, **self.extra) + request = self.factory.get("/", data={"page": 2, "page_size": 1}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - ('; rel="prev", ' - '; rel="next", ' - '; rel="last", ' - '; rel="first"')) + response["Link"], + ( + '; rel="prev", ' + '; rel="next", ' + '; rel="last", ' + '; rel="first"' + ), + ) - request = self.factory.get('/', data={"page_size": "3"}, **self.extra) + request = self.factory.get("/", data={"page_size": "3"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) request = self.factory.get( - '/', data={"page": "1", "page_size": "2"}, **self.extra) + "/", data={"page": "1", "page_size": "2"}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # invalid page returns a 404 - request = self.factory.get('/', data={"page": "invalid"}, **self.extra) + request = self.factory.get("/", data={"page": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 404) # invalid page size is ignored - request = self.factory.get('/', data={"page_size": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"page_size": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # Query param returns correct pagination headers request = self.factory.get( - '/', data={"page_size": "1", "query": "ambulance"}, - **self.extra) + "/", data={"page_size": "1", "query": "ambulance"}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], - ('; rel="next"')) + response["Link"], ('; rel="next"') + ) def test_sort_query_param_with_invalid_values(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # without sort param - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - error_message = ('Expecting property name enclosed in ' - 'double quotes: line 1 column 2 (char 1)') + error_message = ( + "Expecting property name enclosed in " + "double quotes: line 1 column 2 (char 1)" + ) - request = self.factory.get('/', data={"sort": '{'':}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{" ":}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) - request = self.factory.get('/', data={"sort": '{:}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{:}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) - request = self.factory.get('/', data={"sort": '{'':''}'}, - **self.extra) + request = self.factory.get("/", data={"sort": "{" ":" "}"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), error_message) + self.assertEqual(response.data.get("detail"), error_message) # test sort with a key that os likely in the json data - request = self.factory.get('/', data={"sort": 'random'}, - **self.extra) + request = self.factory.get("/", data={"sort": "random"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) def test_data_start_limit_no_records(self): - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no start, limit params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', data={"start": "1", "limit": 2}, - **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": 2}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) def test_data_start_limit(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # no start, limit params - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - self.assertTrue(response.has_header('ETag')) - etag_data = response['Etag'] + self.assertTrue(response.has_header("ETag")) + etag_data = response["Etag"] - request = self.factory.get('/', data={"start": "1", "limit": 2}, - **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": 2}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '9c6f3468-cfda-46e8-84c1-75458e72805d']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "9c6f3468-cfda-46e8-84c1-75458e72805d", + ], + ) - request = self.factory.get('/', data={"start": "3", "limit": 1}, - **self.extra) + request = self.factory.get("/", data={"start": "3", "limit": 1}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['9f0a1508-c3b7-4c99-be00-9b237c26bcbf']) + self.assertEqual( + [i["_uuid"] for i in data], ["9f0a1508-c3b7-4c99-be00-9b237c26bcbf"] + ) - request = self.factory.get('/', data={"limit": "3"}, **self.extra) + request = self.factory.get("/", data={"limit": "3"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] - request = self.factory.get( - '/', data={"start": "1", "limit": "2"}, **self.extra) + request = self.factory.get("/", data={"start": "1", "limit": "2"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertNotEqual(etag_data, response['Etag']) + self.assertNotEqual(etag_data, response["Etag"]) # invalid start is ignored, all data is returned - request = self.factory.get('/', data={"start": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"start": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # invalid limit is ignored, all data is returned - request = self.factory.get('/', data={"limit": "invalid"}, - **self.extra) + request = self.factory.get("/", data={"limit": "invalid"}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # invalid start is ignored, all data is returned - request = self.factory.get('/', data={"start": "", "limit": 10}, - **self.extra) + request = self.factory.get("/", data={"start": "", "limit": 10}, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -664,97 +681,93 @@ def test_paginate_and_sort_streaming_data(self): responses """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # will result in a queryset due to the page and page_size params # hence paging and thus len(self.object_list) for length - query_data = { - "page_size": 3, - "page": 1, - "sort": '{"date_created":-1}' - } - request = self.factory.get('/', data=query_data, - **self.extra) + query_data = {"page_size": 3, "page": 1, "sort": '{"date_created":-1}'} + request = self.factory.get("/", data=query_data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - streaming_data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + streaming_data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(streaming_data), 3) # Test `date_created` field is sorted correctly - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - '-date_created').values_list('id', flat=True)) - items_in_order = [sub.get('_id') for sub in streaming_data] + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("-date_created") + .values_list("id", flat=True) + ) + items_in_order = [sub.get("_id") for sub in streaming_data] self.assertEqual(expected_order[:3], items_in_order) - self.assertTrue(response.has_header('ETag')) + self.assertTrue(response.has_header("ETag")) # Data fetched for page 2 should return a different list - data = { - "page_size": 3, - "page": 2, - "sort": '{"date_created":-1}' - } - request2 = self.factory.get('/', data=data, - **self.extra) + data = {"page_size": 3, "page": 2, "sort": '{"date_created":-1}'} + request2 = self.factory.get("/", data=data, **self.extra) response2 = view(request2, pk=formid) self.assertEqual(response2.status_code, 200) - streaming_data = json.loads(''.join( - [c.decode('utf-8') for c in response2.streaming_content])) + streaming_data = json.loads( + "".join([c.decode("utf-8") for c in response2.streaming_content]) + ) self.assertEqual(len(streaming_data), 1) - pg_2_items_in_order = [sub.get('_id') for sub in streaming_data] + pg_2_items_in_order = [sub.get("_id") for sub in streaming_data] self.assertEqual(expected_order[3:], pg_2_items_in_order) def test_data_start_limit_sort(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk data = {"start": 1, "limit": 2, "sort": '{"_id":1}'} - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - self.assertTrue(response.has_header('ETag')) + self.assertTrue(response.has_header("ETag")) response.render() data = json.loads(response.content) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '9c6f3468-cfda-46e8-84c1-75458e72805d']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "9c6f3468-cfda-46e8-84c1-75458e72805d", + ], + ) def test_filter_pending_submission_reviews(self): """ Test that a user is able to query for null """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) # Create approved submission review for one instance - instances = self.xform.instances.all().order_by('pk') + instances = self.xform.instances.all().order_by("pk") instance = instances[0] self.assertFalse(instance.has_a_review) - self._create_user_and_login('bob', '1234') - ManagerRole.add( - self.user, - self.xform.instances.all().order_by('pk')[0].xform) + self._create_user_and_login("bob", "1234") + ManagerRole.add(self.user, self.xform.instances.all().order_by("pk")[0].xform) data = { - 'note': "Approved!", + "note": "Approved!", "instance": instance.id, - "status": SubmissionReview.APPROVED - } + "status": SubmissionReview.APPROVED, + } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -762,15 +775,13 @@ def test_filter_pending_submission_reviews(self): # Confirm xform submission review enabled self.assertTrue(instance.has_a_review) # Confirm instance json now has _review_status field - self.assertIn('_review_status', dict(instance.json)) + self.assertIn("_review_status", dict(instance.json)) # Confirm instance submission review status - self.assertEqual('1', instance.json['_review_status']) + self.assertEqual("1", instance.json["_review_status"]) # Query xform data by submission review status 3 query_str = '{"_review_status": 3}' - request = self.factory.get( - f'/?query={query_str}', - **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) @@ -779,9 +790,7 @@ def test_filter_pending_submission_reviews(self): # Query xform data by NULL submission review query_str = '{"_review_status": null}' - request = self.factory.get( - f'/?query={query_str}', - **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 3) @@ -789,26 +798,30 @@ def test_filter_pending_submission_reviews(self): @override_settings(STREAM_DATA=True) def test_data_start_limit_sort_json_field(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk # will result in a generator due to the JSON sort # hence self.total_count will be used for length in streaming response data = { "start": 1, "limit": 2, - "sort": '{"transport/available_transportation_types_to_referral_facility":1}' # noqa + "sort": '{"transport/available_transportation_types_to_referral_facility":1}', # noqa } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertTrue(response.has_header('ETag')) - data = json.loads(''.join([ - c.decode('utf-8') for c in response.streaming_content])) + self.assertTrue(response.has_header("ETag")) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 2) - self.assertEqual([i['_uuid'] for i in data], - ['f3d8dc65-91a6-4d0f-9e97-802128083390', - '5b2cc313-fc09-437e-8149-fcd32f695d41']) + self.assertEqual( + [i["_uuid"] for i in data], + [ + "f3d8dc65-91a6-4d0f-9e97-802128083390", + "5b2cc313-fc09-437e-8149-fcd32f695d41", + ], + ) # will result in a queryset due to the page and page_size params # hence paging and thus len(self.object_list) for length @@ -816,30 +829,30 @@ def test_data_start_limit_sort_json_field(self): "page": 1, "page_size": 2, } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 2) data = { "page": 1, "page_size": 3, } - request = self.factory.get('/', data=data, - **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = json.loads(''.join( - [c.decode('utf-8') for c in response.streaming_content])) + data = json.loads( + "".join([c.decode("utf-8") for c in response.streaming_content]) + ) self.assertEqual(len(data), 3) def test_data_anon(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) # data not found for anonymous access to private data @@ -858,19 +871,19 @@ def test_data_anon(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, dict) @@ -878,53 +891,53 @@ def test_data_anon(self): def test_data_public(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_public_anon_user(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_user_public(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk='public') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) self.xform.shared_data = True self.xform.save() formid = self.xform.pk data = _data_list(formid) - response = view(request, pk='public') + response = view(request, pk="public") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) def test_data_bad_formid(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -941,14 +954,14 @@ def test_data_bad_formid(self): formid = "INVALID" response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) error_message = "Invalid form ID. It must be a positive integer" - self.assertEqual(str(response.data['detail']), error_message) + self.assertEqual(str(response.data["detail"]), error_message) def test_data_bad_dataid(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -958,119 +971,116 @@ def test_data_bad_dataid(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = 'INVALID' + dataid = "INVALID" data = _data_instance(dataid) - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_filter_by_submission_time_and_submitted_by(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) submission_time = instance.date_created.strftime(MONGO_STRFTIME) - query_str = ('{"_submission_time": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (submission_time, 'bob')) - data = { - 'query': query_str, - 'limit': 2, - 'sort': [] - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_submission_time": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + submission_time, + "bob", + ) + data = {"query": query_str, "limit": 2, "sort": []} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) def test_filter_by_date_modified(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - instance = self.xform.instances.all().order_by('-date_created')[0] + instance = self.xform.instances.all().order_by("-date_created")[0] date_modified = instance.date_modified.isoformat() - query_str = ('{"_date_modified": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (date_modified, 'bob')) - data = { - 'query': query_str - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_date_modified": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + date_modified, + "bob", + ) + data = {"query": query_str} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) expected_count = self.xform.instances.filter( - date_modified__gte=date_modified).count() + date_modified__gte=date_modified + ).count() self.assertEqual(len(response.data), expected_count) def test_filter_by_submission_time_and_submitted_by_with_data_arg(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) submission_time = instance.date_created.strftime(MONGO_STRFTIME) - query_str = ('{"_submission_time": {"$gte": "%s"},' - ' "_submitted_by": "%s"}' % (submission_time, 'bob')) - data = { - 'data': query_str, - 'limit': 2, - 'sort': [] - } - request = self.factory.get('/', data=data, **self.extra) + query_str = '{"_submission_time": {"$gte": "%s"},' ' "_submitted_by": "%s"}' % ( + submission_time, + "bob", + ) + data = {"data": query_str, "limit": 2, "sort": []} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) def test_filter_by_submission_time_date_formats(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - data = {'query': '{"_submission_time":{"$gt":"2018-04-19"}}'} - request = self.factory.get('/', data=data, **self.extra) + data = {"query": '{"_submission_time":{"$gt":"2018-04-19"}}'} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = {'query': '{"_submission_time":{"$gt":"2018-04-19T14:46:32"}}'} - request = self.factory.get('/', data=data, **self.extra) + data = {"query": '{"_submission_time":{"$gt":"2018-04-19T14:46:32"}}'} + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) def test_data_with_query_parameter(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - instance = self.xform.instances.all().order_by('pk')[0] + instance = self.xform.instances.all().order_by("pk")[0] dataid = instance.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) query_str = '{"_id": "%s"}' % dataid - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) submission_time = instance.date_created.strftime(MONGO_STRFTIME) query_str = '{"_submission_time": {"$gte": "%s"}}' % submission_time - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -1088,47 +1098,55 @@ def test_data_with_query_parameter(self): first_datetime = start_time.strftime(MONGO_STRFTIME) second_datetime = start_time + timedelta(days=1, hours=20) - query_str = '{"_submission_time": {"$gte": "'\ - + first_datetime + '", "$lte": "'\ - + second_datetime.strftime(MONGO_STRFTIME) + '"}}' + query_str = ( + '{"_submission_time": {"$gte": "' + + first_datetime + + '", "$lte": "' + + second_datetime.strftime(MONGO_STRFTIME) + + '"}}' + ) - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) query_str = '{"_id: "%s"}' % dataid - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get('detail'), - u"Expecting ':' delimiter: line 1 column 9 (char 8)") + self.assertEqual( + response.data.get("detail"), + "Expecting ':' delimiter: line 1 column 9 (char 8)", + ) - query_str = '{"transport/available_transportation' \ - '_types_to_referral_facility": {"$i": "%s"}}' % "ambula" - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = ( + '{"transport/available_transportation' + '_types_to_referral_facility": {"$i": "%s"}}' % "ambula" + ) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) # search a text - query_str = 'uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = "uuid:9f0a1508-c3b7-4c99-be00-9b237c26bcbf" + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # search an integer query_str = 7545 - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) def test_anon_data_list(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) @@ -1139,55 +1157,54 @@ def test_add_form_tag_propagates_to_data_tags(self): self._make_submissions() xform = XForm.objects.all()[0] pk = xform.id - view = XFormViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) - data_view = DataViewSet.as_view({ - 'get': 'list', - }) + view = XFormViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) + data_view = DataViewSet.as_view( + { + "get": "list", + } + ) # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk) self.assertEqual(response.data, []) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=pk) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, ['hello']) + self.assertEqual(response.data, ["hello"]) for i in self.xform.instances.all(): - self.assertIn('hello', i.tags.names()) + self.assertIn("hello", i.tags.names()) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) # remove tag "hello" - request = self.factory.delete('/', data={"tags": "hello"}, - **self.extra) - response = view(request, pk=pk, label='hello') + request = self.factory.delete("/", data={"tags": "hello"}, **self.extra) + response = view(request, pk=pk, label="hello") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) for i in self.xform.instances.all(): - self.assertNotIn('hello', i.tags.names()) + self.assertNotIn("hello", i.tags.names()) def test_data_tags(self): """Test that when a tag is applied on an xform, @@ -1198,82 +1215,79 @@ def test_data_tags(self): pk = self.xform.pk i = self.xform.instances.all()[0] dataid = i.pk - data_view = DataViewSet.as_view({ - 'get': 'list', - }) - view = DataViewSet.as_view({ - 'get': 'labels', - 'post': 'labels', - 'delete': 'labels' - }) + data_view = DataViewSet.as_view( + { + "get": "list", + } + ) + view = DataViewSet.as_view( + {"get": "labels", "post": "labels", "delete": "labels"} + ) # no tags - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), submission_count) # add tag "hello" - request = self.factory.post('/', data={"tags": "hello"}, **self.extra) + request = self.factory.post("/", data={"tags": "hello"}, **self.extra) response = view(request, pk=pk, dataid=dataid) self.assertEqual(response.status_code, 201) - self.assertEqual(response.data, ['hello']) - self.assertIn('hello', Instance.objects.get(pk=dataid).tags.names()) + self.assertEqual(response.data, ["hello"]) + self.assertIn("hello", Instance.objects.get(pk=dataid).tags.names()) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=pk, dataid=dataid) - self.assertEqual(response.data, ['hello']) + self.assertEqual(response.data, ["hello"]) - request = self.factory.get('/', {'tags': 'hello'}, **self.extra) + request = self.factory.get("/", {"tags": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - request = self.factory.get('/', {'not_tagged': 'hello'}, **self.extra) + request = self.factory.get("/", {"not_tagged": "hello"}, **self.extra) response = data_view(request, pk=pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), submission_count - 1) # remove tag "hello" - request = self.factory.delete('/', **self.extra) - response = view(request, pk=pk, dataid=dataid, label='hello') + request = self.factory.delete("/", **self.extra) + response = view(request, pk=pk, dataid=dataid, label="hello") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - self.assertNotIn( - 'hello', Instance.objects.get(pk=dataid).tags.names()) + self.assertNotIn("hello", Instance.objects.get(pk=dataid).tags.names()) def test_labels_action_with_params(self): self._make_submissions() xform = XForm.objects.all()[0] pk = xform.id dataid = xform.instances.all()[0].id - view = DataViewSet.as_view({ - 'get': 'labels' - }) + view = DataViewSet.as_view({"get": "labels"}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=pk, dataid=dataid, label='hello') + request = self.factory.get("/", **self.extra) + response = view(request, pk=pk, dataid=dataid, label="hello") self.assertEqual(response.status_code, 200) def test_data_list_filter_by_user(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk bobs_data = _data_list(formid)[0] previous_user = self.user - self._create_user_and_login('alice', 'alice') - self.assertEqual(self.user.username, 'alice') + self._create_user_and_login("alice", "alice") + self.assertEqual(self.user.username, "alice") self.assertNotEqual(previous_user, self.user) ReadOnlyRole.add(self.user, self.xform) @@ -1281,73 +1295,72 @@ def test_data_list_filter_by_user(self): # publish alice's form self._publish_transportation_form() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} formid = self.xform.pk alice_data = _data_list(formid)[0] - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) # should be both bob's and alice's form self.assertEqual( - sorted(response.data, key=lambda x: x['id']), - sorted([bobs_data, alice_data], key=lambda x: x['id'])) + sorted(response.data, key=lambda x: x["id"]), + sorted([bobs_data, alice_data], key=lambda x: x["id"]), + ) # apply filter, see only bob's forms - request = self.factory.get('/', data={'owner': 'bob'}, **self.extra) + request = self.factory.get("/", data={"owner": "bob"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [bobs_data]) # apply filter, see only alice's forms - request = self.factory.get('/', data={'owner': 'alice'}, **self.extra) + request = self.factory.get("/", data={"owner": "alice"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [alice_data]) # apply filter, see a non existent user - request = self.factory.get('/', data={'owner': 'noone'}, **self.extra) + request = self.factory.get("/", data={"owner": "noone"}, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_get_enketo_edit_url(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'enketo'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "enketo"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) # add data check - self.assertEqual( - response.data, - {'detail': 'return_url not provided.'}) + self.assertEqual(response.data, {"detail": "return_url not provided."}) request = self.factory.get( - '/', - data={'return_url': "http://test.io/test_url"}, **self.extra) + "/", data={"return_url": "http://test.io/test_url"}, **self.extra + ) with HTTMock(enketo_edit_mock): response = view(request, pk=formid, dataid=dataid) self.assertEqual( - response.data['url'], + response.data["url"], "https://hmh2a.enketo.ona.io/edit/XA0bG8Df?instance_id=" "672927e3-9ad4-42bb-9538-388ea1fb6699&returnUrl=http://test" - ".io/test_url") + ".io/test_url", + ) with HTTMock(enketo_mock_http_413): response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) - self.assertEqual(response.get('Cache-Control'), None) + self.assertEqual(response.get("Cache-Control"), None) def test_get_form_public_data(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) @@ -1362,25 +1375,26 @@ def test_get_form_public_data(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) # access to a public data as other user - self._create_user_and_login('alice', 'alice') - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.get('/', **self.extra) + self._create_user_and_login("alice", "alice") + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = _data_instance(dataid) self.assertDictContainsSubset( - data, sorted(response.data, key=lambda x: x['_id'])[0]) + data, sorted(response.data, key=lambda x: x["_id"])[0] + ) def test_same_submission_with_different_attachments(self): """ @@ -1394,90 +1408,104 @@ def test_same_submission_with_different_attachments(self): """ xform = self._publish_markdown(images_md, self.user) submission_file = NamedTemporaryFile(delete=False) - with open(submission_file.name, 'w') as xml_file: + with open(submission_file.name, "w") as xml_file: xml_file.write( "" "1335783522563.jpg" "1442323232322.jpg" "uuid:729f173c688e482486a48661700455ff" - "" % - (xform.id_string)) + "    " % (xform.id_string) + ) media_file = "1335783522563.jpg" self._make_submission_w_attachment( submission_file.name, - os.path.join(self.this_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], media_file)) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + media_file, + ), + ) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = xform.pk data = { - 'id': formid, - 'id_string': xform.id_string, - 'title': xform.title, - 'description': '', - 'url': 'http://testserver/api/v1/data/%s' % formid + "id": formid, + "id_string": xform.id_string, + "title": xform.title, + "description": "", + "url": "http://testserver/api/v1/data/%s" % formid, } self.assertEqual(response.data[1], data) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - instance = xform.instances.all().order_by('id')[0] + instance = xform.instances.all().order_by("id")[0] dataid = instance.pk attachment = instance.attachments.all().first() data = { - '_bamboo_dataset_id': '', - '_attachments': [{ - 'download_url': get_attachment_url(attachment), - 'small_download_url': - get_attachment_url(attachment, 'small'), - 'medium_download_url': - get_attachment_url(attachment, 'medium'), - 'mimetype': attachment.mimetype, - 'instance': attachment.instance.pk, - 'filename': attachment.media_file.name, - 'name': attachment.name, - 'id': attachment.pk, - 'xform': xform.id} + "_bamboo_dataset_id": "", + "_attachments": [ + { + "download_url": get_attachment_url(attachment), + "small_download_url": get_attachment_url(attachment, "small"), + "medium_download_url": get_attachment_url(attachment, "medium"), + "mimetype": attachment.mimetype, + "instance": attachment.instance.pk, + "filename": attachment.media_file.name, + "name": attachment.name, + "id": attachment.pk, + "xform": xform.id, + } ], - '_geolocation': [None, None], - '_xform_id_string': xform.id_string, - '_status': 'submitted_via_web', - '_id': dataid, - 'image1': '1335783522563.jpg' + "_geolocation": [None, None], + "_xform_id_string": xform.id_string, + "_status": "submitted_via_web", + "_id": dataid, + "image1": "1335783522563.jpg", } self.assertDictContainsSubset(data, sorted(response.data)[0]) - patch_value = 'onadata.libs.utils.logger_tools.get_filtered_instances' + patch_value = "onadata.libs.utils.logger_tools.get_filtered_instances" with patch(patch_value) as get_filtered_instances: get_filtered_instances.return_value = Instance.objects.filter( - uuid='#doesnotexist') + uuid="#doesnotexist" + ) media_file = "1442323232322.jpg" self._make_submission_w_attachment( submission_file.name, - os.path.join(self.this_directory, 'fixtures', 'transportation', - 'instances', self.surveys[0], media_file)) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + media_file, + ), + ) attachment = Attachment.objects.get(name=media_file) - data['_attachments'] = data.get('_attachments') + [{ - 'download_url': get_attachment_url(attachment), - 'small_download_url': - get_attachment_url(attachment, 'small'), - 'medium_download_url': - get_attachment_url(attachment, 'medium'), - 'mimetype': attachment.mimetype, - 'instance': attachment.instance.pk, - 'filename': attachment.media_file.name, - 'name': attachment.name, - 'id': attachment.pk, - 'xform': xform.id - }] + data["_attachments"] = data.get("_attachments") + [ + { + "download_url": get_attachment_url(attachment), + "small_download_url": get_attachment_url(attachment, "small"), + "medium_download_url": get_attachment_url(attachment, "medium"), + "mimetype": attachment.mimetype, + "instance": attachment.instance.pk, + "filename": attachment.media_file.name, + "name": attachment.name, + "id": attachment.pk, + "xform": xform.id, + } + ] self.maxDiff = None response = view(request, pk=formid) - self.assertDictContainsSubset(sorted([data])[0], - sorted(response.data)[0]) + self.assertDictContainsSubset(sorted([data])[0], sorted(response.data)[0]) self.assertEqual(response.status_code, 200) submission_file.close() os.unlink(submission_file.name) @@ -1485,8 +1513,8 @@ def test_same_submission_with_different_attachments(self): def test_data_w_attachment(self): self._submit_transport_instance_w_attachment() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk @@ -1496,63 +1524,60 @@ def test_data_w_attachment(self): self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) self.assertTrue(self.xform.instances.count()) - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk data = { - '_bamboo_dataset_id': '', - '_attachments': [{ - 'download_url': get_attachment_url(self.attachment), - 'small_download_url': - get_attachment_url(self.attachment, 'small'), - 'medium_download_url': - get_attachment_url(self.attachment, 'medium'), - 'mimetype': self.attachment.mimetype, - 'instance': self.attachment.instance.pk, - 'filename': self.attachment.media_file.name, - 'name': self.attachment.name, - 'id': self.attachment.pk, - 'xform': self.xform.id} + "_bamboo_dataset_id": "", + "_attachments": [ + { + "download_url": get_attachment_url(self.attachment), + "small_download_url": get_attachment_url(self.attachment, "small"), + "medium_download_url": get_attachment_url( + self.attachment, "medium" + ), + "mimetype": self.attachment.mimetype, + "instance": self.attachment.instance.pk, + "filename": self.attachment.media_file.name, + "name": self.attachment.name, + "id": self.attachment.pk, + "xform": self.xform.id, + } ], - '_geolocation': [None, None], - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_status': 'submitted_via_web', - '_id': dataid + "_geolocation": [None, None], + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_status": "submitted_via_web", + "_id": dataid, } self.assertDictContainsSubset(data, sorted(response.data)[0]) data = { - '_xform_id_string': 'transportation_2011_07_25', - 'transport/available_transportation_types_to_referral_facility': - 'none', - '_submitted_by': 'bob', + "_xform_id_string": "transportation_2011_07_25", + "transport/available_transportation_types_to_referral_facility": "none", + "_submitted_by": "bob", } - view = DataViewSet.as_view({'get': 'retrieve'}) + view = DataViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # get the first xform instance returned first_xform_instance = self.xform.instances.filter(pk=dataid) self.assertEqual(first_xform_instance[0].deleted_by, None) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) first_xform_instance = self.xform.instances.filter(pk=dataid) @@ -1560,65 +1585,66 @@ def test_delete_submission(self, send_message_mock): # message sent upon delete self.assertTrue(send_message_mock.called) send_message_mock.assert_called_with( - instance_id=dataid, target_id=formid, target_type=XFORM, - user=request.user, message_verb=SUBMISSION_DELETED) + instance_id=dataid, + target_id=formid, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, + ) # second delete of same submission should return 404 - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 404) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Managers can delete role.ManagerRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 2) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') - @patch('onadata.apps.viewer.signals._post_process_submissions') - def test_post_save_signal_on_submission_deletion(self, mock, - send_message_mock): + @patch("onadata.apps.api.viewsets.data_viewset.send_message") + @patch("onadata.apps.viewer.signals._post_process_submissions") + def test_post_save_signal_on_submission_deletion(self, mock, send_message_mock): # test that post_save_submission signal is sent # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) self.assertEqual(self.response.status_code, 201) formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - request = self.factory.delete('/', **self.extra) + dataid = self.xform.instances.all().order_by("id")[0].pk + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) @@ -1632,8 +1658,7 @@ def test_post_save_signal_on_submission_deletion(self, mock, self.assertEqual(mock.call_count, 1) self.assertTrue(send_message_mock.called) - @patch( - 'onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_deletion_of_bulk_submissions(self, send_message_mock): self._make_submissions() @@ -1643,18 +1668,15 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): self.assertEqual(initial_count, 4) self.assertEqual(self.xform.num_of_submissions, 4) - view = DataViewSet.as_view({'delete': 'destroy'}) + view = DataViewSet.as_view({"delete": "destroy"}) # test with invalid instance id's data = {"instance_ids": "john,doe"} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - self.assertEqual( - response.data.get('detail'), - u"Invalid data ids were provided." - ) + self.assertEqual(response.data.get("detail"), "Invalid data ids were provided.") self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() self.assertEqual(current_count, initial_count) @@ -1663,37 +1685,43 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): # test with valid instance id's records_to_be_deleted = self.xform.instances.all()[:2] - instance_ids = ','.join([str(i.pk) for i in records_to_be_deleted]) + instance_ids = ",".join([str(i.pk) for i in records_to_be_deleted]) data = {"instance_ids": instance_ids} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual( - response.data.get('message'), - "%d records were deleted" % len(records_to_be_deleted) + response.data.get("message"), + "%d records were deleted" % len(records_to_be_deleted), ) self.assertTrue(send_message_mock.called) send_message_mock.called_with( - [str(i.pk) for i in records_to_be_deleted], formid, XFORM, - request.user, SUBMISSION_DELETED) + [str(i.pk) for i in records_to_be_deleted], + formid, + XFORM, + request.user, + SUBMISSION_DELETED, + ) self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() self.assertNotEqual(current_count, initial_count) self.assertEqual(current_count, 2) self.assertEqual(self.xform.num_of_submissions, 2) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_inactive_form(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view( + { + "delete": "destroy", + } + ) - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) @@ -1702,27 +1730,34 @@ def test_delete_submission_inactive_form(self, send_message_mock): self.xform.downloadable = False self.xform.save() - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 400) self.assertTrue(send_message_mock.called) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submissions(self, send_message_mock): xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # Add multiple submissions for x in range(1, 11): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self.xform.refresh_from_db() @@ -1731,23 +1766,27 @@ def test_delete_submissions(self, send_message_mock): self.assertEqual(initial_count, 9) self.assertEqual(self.xform.num_of_submissions, 9) - view = DataViewSet.as_view({'delete': 'destroy'}) + view = DataViewSet.as_view({"delete": "destroy"}) deleted_instances_subset = self.xform.instances.all()[:6] - instance_ids = ','.join([str(i.pk) for i in deleted_instances_subset]) - data = {"instance_ids": instance_ids, 'delete_all': False} + instance_ids = ",".join([str(i.pk) for i in deleted_instances_subset]) + data = {"instance_ids": instance_ids, "delete_all": False} - request = self.factory.delete('/', data=data, **self.extra) + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual( - response.data.get('message'), - "%d records were deleted" % len(deleted_instances_subset) + response.data.get("message"), + "%d records were deleted" % len(deleted_instances_subset), ) self.assertTrue(send_message_mock.called) send_message_mock.called_with( - [str(i.pk) for i in deleted_instances_subset], formid, XFORM, - request.user, SUBMISSION_DELETED) + [str(i.pk) for i in deleted_instances_subset], + formid, + XFORM, + request.user, + SUBMISSION_DELETED, + ) # Test that num of submissions for the form is successfully updated self.xform.refresh_from_db() @@ -1757,92 +1796,80 @@ def test_delete_submissions(self, send_message_mock): self.assertEqual(self.xform.num_of_submissions, 3) # Test delete_all submissions for the form - data = {'delete_all': True} - request = self.factory.delete('/', data=data, **self.extra) + data = {"delete_all": True} + request = self.factory.delete("/", data=data, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertEqual( - response.data.get('message'), - "3 records were deleted") + self.assertEqual(response.data.get("message"), "3 records were deleted") # Test project details updated successfully self.assertEqual( self.xform.project.date_modified.strftime("%Y-%m-%d %H:%M:%S"), - timezone.now().strftime("%Y-%m-%d %H:%M:%S")) + timezone.now().strftime("%Y-%m-%d %H:%M:%S"), + ) # Test XForm now contains no submissions self.xform.refresh_from_db() - delete_all_current_count = self.xform.instances.filter( - deleted_at=None).count() + delete_all_current_count = self.xform.instances.filter(deleted_at=None).count() self.assertNotEqual(current_count, delete_all_current_count) self.assertEqual(delete_all_current_count, 0) self.assertEqual(self.xform.num_of_submissions, 0) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_by_editor(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Editor can delete submission role.EditorRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) self.assertTrue(send_message_mock.called) - @patch('onadata.apps.api.viewsets.data_viewset.send_message') + @patch("onadata.apps.api.viewsets.data_viewset.send_message") def test_delete_submission_by_owner(self, send_message_mock): self._make_submissions() formid = self.xform.pk - dataid = self.xform.instances.all().order_by('id')[0].pk - view = DataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'list' - }) + dataid = self.xform.instances.all().order_by("id")[0].pk + view = DataViewSet.as_view({"delete": "destroy", "get": "list"}) # 4 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 4) - self._create_user_and_login(username='alice', password='alice') + self._create_user_and_login(username="alice", password="alice") # Owner can delete submission role.OwnerRole.add(self.user, self.xform) - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} - request = self.factory.delete('/', **self.extra) - dataid = self.xform.instances.filter(deleted_at=None)\ - .order_by('id')[0].pk + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} + request = self.factory.delete("/", **self.extra) + dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk response = view(request, pk=formid, dataid=dataid) self.assertEqual(response.status_code, 204) # remaining 3 submissions - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(len(response.data), 3) self.assertTrue(send_message_mock.called) @@ -1850,142 +1877,110 @@ def test_delete_submission_by_owner(self, send_message_mock): def test_geojson_format(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - view = DataViewSet.as_view({'get': 'retrieve'}) - data_get = { - "fields": 'today' - } - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + data_get = {"fields": "today"} + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) self.assertEqual(self.xform.instances.count(), 4) test_geo = { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], }, - 'properties': { - 'id': dataid, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } + "properties": {"id": dataid, "xform": self.xform.pk, "today": "2015-01-15"}, } self.assertEqual(response.data, test_geo) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') - instances = self.xform.instances.all().order_by('id') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") + instances = self.xform.instances.all().order_by("id") data = { - 'type': 'FeatureCollection', - 'features': [ + "type": "FeatureCollection", + "features": [ { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[0].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[0].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[1].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[1].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], + }, + "properties": { + "id": instances[2].pk, + "xform": self.xform.pk, + "today": "2015-01-15", }, - 'properties': { - 'id': instances[2].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } }, { - 'type': 'Feature', - 'geometry': { - 'type': 'GeometryCollection', - 'geometries': [{ - 'type': 'Point', - 'coordinates': [ - 36.787219, - -1.294197 - ] - } - ] + "type": "Feature", + "geometry": { + "type": "GeometryCollection", + "geometries": [ + {"type": "Point", "coordinates": [36.787219, -1.294197]} + ], }, - 'properties': { - 'id': instances[3].pk, - 'xform': self.xform.pk, - 'today': '2015-01-15' - } - } - ] + "properties": { + "id": instances[3].pk, + "xform": self.xform.pk, + "today": "2015-01-15", + }, + }, + ], } self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) - @patch( - 'onadata.apps.api.viewsets.data_viewset' - '.DataViewSet.paginate_queryset') + @patch("onadata.apps.api.viewsets.data_viewset" ".DataViewSet.paginate_queryset") def test_retry_on_operational_error(self, mock_paginate_queryset): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk mock_paginate_queryset.side_effect = [ OperationalError, - Instance.objects.filter(xform_id=formid)[:2]] + Instance.objects.filter(xform_id=formid)[:2], + ] - request = self.factory.get('/', data={"page": "1", "page_size": 2}, - **self.extra) + request = self.factory.get( + "/", data={"page": "1", "page_size": 2}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(mock_paginate_queryset.call_count, 2) @@ -1993,120 +1988,102 @@ def test_retry_on_operational_error(self, mock_paginate_queryset): def test_geojson_geofield(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'location', - "fields": 'today' - } + data_get = {"geo_field": "location", "fields": "today"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) test_loc = geojson.Feature( - geometry=geojson.GeometryCollection([ - geojson.Point((36.787219, -1.294197))]), - properties={ - 'xform': self.xform.pk, - 'id': dataid, - 'today': '2015-01-15' - } + geometry=geojson.GeometryCollection( + [geojson.Point((36.787219, -1.294197))] + ), + properties={"xform": self.xform.pk, "id": dataid, "today": "2015-01-15"}, ) - if 'id' in test_loc: - test_loc.pop('id') + if "id" in test_loc: + test_loc.pop("id") self.assertEqual(response.data, test_loc) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['type'], 'FeatureCollection') - self.assertEqual(len(response.data['features']), 4) - self.assertEqual(response.data['features'][0]['type'], 'Feature') + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") self.assertEqual( - response.data['features'][0]['geometry']['geometries'][0]['type'], - 'Point' + response.data["features"][0]["geometry"]["geometries"][0]["type"], "Point" ) def test_geojson_linestring(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'path', - "fields": 'today,path' - } + data_get = {"geo_field": "path", "fields": "today,path"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['type'], 'Feature') - self.assertEqual(len(response.data['geometry']['coordinates']), 5) - self.assertIn('path', response.data['properties']) - self.assertEqual(response.data['geometry']['type'], 'LineString') + self.assertEqual(response.data["type"], "Feature") + self.assertEqual(len(response.data["geometry"]["coordinates"]), 5) + self.assertIn("path", response.data["properties"]) + self.assertEqual(response.data["geometry"]["type"], "LineString") - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['type'], 'FeatureCollection') - self.assertEqual(len(response.data['features']), 4) - self.assertEqual(response.data['features'][0]['type'], 'Feature') - self.assertEqual(response.data['features'][0]['geometry']['type'], - 'LineString') + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") + self.assertEqual(response.data["features"][0]["geometry"]["type"], "LineString") def test_geojson_polygon(self): self._publish_submit_geojson() - dataid = self.xform.instances.all().order_by('id')[0].pk + dataid = self.xform.instances.all().order_by("id")[0].pk - data_get = { - "geo_field": 'shape', - "fields": 'today,shape' - } + data_get = {"geo_field": "shape", "fields": "today,shape"} - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, dataid=dataid, - format='geojson') + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, dataid=dataid, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['type'], 'Feature') - self.assertEqual(len(response.data['geometry']['coordinates'][0]), 6) - self.assertIn('shape', response.data['properties']) - self.assertEqual(response.data['geometry']['type'], 'Polygon') + self.assertEqual(response.data["type"], "Feature") + self.assertEqual(len(response.data["geometry"]["coordinates"][0]), 6) + self.assertIn("shape", response.data["properties"]) + self.assertEqual(response.data["geometry"]["type"], "Polygon") - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', data=data_get, **self.extra) - response = view(request, pk=self.xform.pk, format='geojson') + request = self.factory.get("/", data=data_get, **self.extra) + response = view(request, pk=self.xform.pk, format="geojson") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['type'], 'FeatureCollection') - self.assertEqual(len(response.data['features']), 4) - self.assertEqual(response.data['features'][0]['type'], 'Feature') - self.assertEqual(response.data['features'][0]['geometry']['type'], - 'Polygon') + self.assertEqual(response.data["type"], "FeatureCollection") + self.assertEqual(len(response.data["features"]), 4) + self.assertEqual(response.data["features"][0]["type"], "Feature") + self.assertEqual(response.data["features"][0]["geometry"]["type"], "Polygon") def test_data_in_public_project(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk with HTTMock(enketo_urls_mock): response = view(request, pk=formid) @@ -2115,15 +2092,14 @@ def test_data_in_public_project(self): # get project id projectid = self.xform.project.pk - view = ProjectViewSet.as_view({ - 'put': 'update' - }) + view = ProjectViewSet.as_view({"put": "update"}) - data = {'public': True, - 'name': 'test project', - 'owner': 'http://testserver/api/v1/users/%s' - % self.user.username} - request = self.factory.put('/', data=data, **self.extra) + data = { + "public": True, + "name": "test project", + "owner": "http://testserver/api/v1/users/%s" % self.user.username, + } + request = self.factory.put("/", data=data, **self.extra) response = view(request, pk=projectid) self.assertEqual(response.status_code, 200) @@ -2131,8 +2107,8 @@ def test_data_in_public_project(self): self.assertEqual(self.xform.shared, True) # anonymous user - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") formid = self.xform.pk response = view(request, pk=formid) @@ -2146,32 +2122,40 @@ def test_data_diff_version(self): self.xform.save() # make more submission after form update - surveys = ['transport_2011-07-25_19-05-36-edited'] + surveys = ["transport_2011-07-25_19-05-36-edited"] main_directory = os.path.dirname(main_tests.__file__) - paths = [os.path.join(main_directory, 'fixtures', 'transportation', - 'instances_w_uuid', s, s + '.xml') - for s in surveys] + paths = [ + os.path.join( + main_directory, + "fixtures", + "transportation", + "instances_w_uuid", + s, + s + ".xml", + ) + for s in surveys + ] - auth = DigestAuth('bob', 'bob') + auth = DigestAuth("bob", "bob") for path in paths: self._make_submission(path, None, None, auth=auth) - data_view = DataViewSet.as_view({'get': 'list'}) + data_view = DataViewSet.as_view({"get": "list"}) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = data_view(request, pk=self.xform.pk) self.assertEqual(len(response.data), 5) query_str = '{"_version": "2014111"}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) self.assertEqual(len(response.data), 4) query_str = '{"_version": "212121211"}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = data_view(request, pk=self.xform.pk) self.assertEqual(len(response.data), 1) @@ -2179,52 +2163,55 @@ def test_data_diff_version(self): def test_last_modified_on_data_list_response(self): self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertEqual(response.get('Cache-Control'), 'max-age=60') + self.assertEqual(response.get("Cache-Control"), "max-age=60") - self.assertTrue(response.has_header('ETag')) - etag_value = response.get('ETag') + self.assertTrue(response.has_header("ETag")) + etag_value = response.get("ETag") - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertEqual(etag_value, response.get('ETag')) + self.assertEqual(etag_value, response.get("ETag")) # delete one submission inst = Instance.objects.filter(xform=self.xform) inst[0].delete() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - self.assertNotEqual(etag_value, response.get('ETag')) + self.assertNotEqual(etag_value, response.get("ETag")) def test_submission_history(self): """Test submission json includes has_history key""" # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -2235,27 +2222,32 @@ def test_submission_history(self): # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) self.assertEqual(instance_count, Instance.objects.count()) - self.assertEqual(instance_history_count + 1, - InstanceHistory.objects.count()) + self.assertEqual(instance_history_count + 1, InstanceHistory.objects.count()) # retrieve submission history - view = DataViewSet.as_view({'get': 'history'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "history"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid=instance.id) self.assertEqual(response.status_code, 200) @@ -2268,12 +2260,11 @@ def test_submission_history(self): def test_submission_history_not_digit(self): """Test submission json includes has_history key""" # retrieve submission history - view = DataViewSet.as_view({'get': 'history'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "history"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid="boo!") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['detail'], - 'Data ID should be an integer') + self.assertEqual(response.data["detail"], "Data ID should be an integer") history_instance_count = InstanceHistory.objects.count() self.assertEqual(history_instance_count, 0) @@ -2283,13 +2274,13 @@ def test_data_endpoint_etag_on_submission_edit(self): # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) def _data_response(): - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) @@ -2300,45 +2291,54 @@ def _data_response(): # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) response = _data_response() - etag_data = response['Etag'] + etag_data = response["Etag"] # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) response = _data_response() - self.assertNotEqual(etag_data, response['Etag']) - etag_data = response['Etag'] + self.assertNotEqual(etag_data, response["Etag"]) + etag_data = response["Etag"] response = _data_response() - self.assertEqual(etag_data, response['Etag']) + self.assertEqual(etag_data, response["Etag"]) def test_submission_edit_w_blank_field(self): """Test submission json includes has_history key""" # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) # create submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -2348,11 +2348,14 @@ def test_submission_edit_w_blank_field(self): # edit submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited_blank.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited_blank.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) @@ -2360,12 +2363,12 @@ def test_submission_edit_w_blank_field(self): self.assertEqual(instance_count, Instance.objects.count()) # retrieve submission history - view = DataViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.xform.pk, dataid=instance.id) self.assertEqual(instance_count, Instance.objects.count()) - self.assertNotIn('name', response.data) + self.assertNotIn("name", response.data) def test_filterset(self): # create submissions to test with @@ -2374,9 +2377,9 @@ def test_filterset(self): # the original count of Instance objects instance_count = Instance.objects.all().count() # ## Test no filters - request = self.factory.get('/', **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), instance_count) # ## Test version # all the instances created have no version @@ -2385,23 +2388,25 @@ def test_filterset(self): instance = Instance.objects.last() instance.version = 777 instance.save() - request = self.factory.get('/', {'version': 777}, **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') - self.assertEqual(len(response.data), - Instance.objects.filter(version=777).count()) + request = self.factory.get("/", {"version": 777}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") + self.assertEqual( + len(response.data), Instance.objects.filter(version=777).count() + ) # ## Test Status # all the instanced created have the same status i.e. # 'submitted_via_web' . We now set one instance to have a different # status and filter for it instance = Instance.objects.last() - instance.status = 'fortytwo' + instance.status = "fortytwo" instance.save() - request = self.factory.get('/', {'status': 'fortytwo'}, **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') - self.assertEqual(len(response.data), - Instance.objects.filter(status='fortytwo').count()) + request = self.factory.get("/", {"status": "fortytwo"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") + self.assertEqual( + len(response.data), Instance.objects.filter(status="fortytwo").count() + ) # ## Test date_created # all the instances created have the same date_created i.e. the # datetime at the time of creation @@ -2411,15 +2416,14 @@ def test_filterset(self): initial_year = instance.date_created.year instance.date_created = instance.date_created - one_year instance.save() - request = self.factory.get('/', - {'date_created__year__lt': initial_year}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"date_created__year__lt": initial_year}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - date_created__year__lt=initial_year).count() + Instance.objects.filter(date_created__year__lt=initial_year).count(), ) # ## Test last_edited # all the instances created have None as the last_edited @@ -2427,73 +2431,66 @@ def test_filterset(self): # we can filter for this one instance instance.last_edited = instance.date_created - one_year instance.save() - request = self.factory.get('/', - {'last_edited__year': initial_year}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"last_edited__year": initial_year}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - last_edited__year=initial_year).count() + Instance.objects.filter(last_edited__year=initial_year).count(), ) # ## Test uuid # all the instances created have different uuid values # we test this by looking for just one match - request = self.factory.get('/', - {'uuid': instance.uuid}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"uuid": instance.uuid}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( - len(response.data), - Instance.objects.filter(uuid=instance.uuid).count() + len(response.data), Instance.objects.filter(uuid=instance.uuid).count() ) # ## Test user # all the forms are owned by a user named bob # we create a user named alice and confirm that data filtered for her # has a count fo 0 - user_alice = self._create_user('alice', 'alice') - request = self.factory.get('/', - {'user__id': user_alice.id}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + user_alice = self._create_user("alice", "alice") + request = self.factory.get("/", {"user__id": user_alice.id}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 0) # we make one instance belong to user_alice and then filter for that instance.user = user_alice instance.save() - request = self.factory.get('/', - {'user__username': user_alice.username}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"user__username": user_alice.username}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter(user__username=user_alice.username).count() + Instance.objects.filter(user__username=user_alice.username).count(), ) # ## Test submitted_by # submitted_by is mapped to the user field # to test, we do the same as we did for user above - user_mosh = self._create_user('mosh', 'mosh') - request = self.factory.get('/', - {'submitted_by__id': user_mosh.id}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + user_mosh = self._create_user("mosh", "mosh") + request = self.factory.get( + "/", {"submitted_by__id": user_mosh.id}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 0) # we make one instance belong to user_mosh and then filter for that instance.user = user_mosh instance.save() - request = self.factory.get('/', - {'submitted_by__username': - user_mosh.username}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"submitted_by__username": user_mosh.username}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter(user__username=user_mosh.username).count() + Instance.objects.filter(user__username=user_mosh.username).count(), ) # ## Test survey_type # all the instances created have the same survey_type @@ -2501,32 +2498,27 @@ def test_filterset(self): new_survey_type = SurveyType.objects.create(slug="hunter2") instance.survey_type = new_survey_type instance.save() - request = self.factory.get('/', - {'survey_type__slug': new_survey_type.slug}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get( + "/", {"survey_type__slug": new_survey_type.slug}, **self.extra + ) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual( len(response.data), - Instance.objects.filter( - survey_type__slug=new_survey_type.slug).count() + Instance.objects.filter(survey_type__slug=new_survey_type.slug).count(), ) # ## Test all_media_received # all the instances have media_all_received == True - request = self.factory.get('/', - {'media_all_received': 'false'}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"media_all_received": "false"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 1) # we set one to False and filter for it instance.media_all_received = False instance.save() - request = self.factory.get('/', - {'media_all_received': 'false'}, - **self.extra) - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='json') + request = self.factory.get("/", {"media_all_received": "false"}, **self.extra) + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="json") self.assertEqual(len(response.data), 2) def test_floip_format(self): @@ -2534,26 +2526,27 @@ def test_floip_format(self): Test FLOIP output results. """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk request = self.factory.get( - '/', - HTTP_ACCEPT='application/vnd.org.flowinterop.results+json', - **self.extra) + "/", + HTTP_ACCEPT="application/vnd.org.flowinterop.results+json", + **self.extra, + ) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) response.render() floip_list = json.loads(response.content) self.assertTrue(isinstance(floip_list, list)) - floip_row = [x for x in floip_list if x[-2] == 'none'][0] - self.assertEqual(floip_row[0], - response.data[0]['_submission_time'] + '+00:00') - self.assertEqual(floip_row[2], 'bob') - self.assertEqual(floip_row[3], response.data[0]['_uuid']) + floip_row = [x for x in floip_list if x[-2] == "none"][0] + self.assertEqual(floip_row[0], response.data[0]["_submission_time"] + "+00:00") + self.assertEqual(floip_row[2], "bob") + self.assertEqual(floip_row[3], response.data[0]["_uuid"]) self.assertEqual( floip_row[4], - 'transport/available_transportation_types_to_referral_facility') - self.assertEqual(floip_row[5], 'none') + "transport/available_transportation_types_to_referral_facility", + ) + self.assertEqual(floip_row[5], "none") def test_submission_count_for_day_tracking(self): """ @@ -2567,8 +2560,7 @@ def test_submission_count_for_day_tracking(self): c_date = datetime.datetime.today() instances = Instance.objects.filter(date_created__date=c_date.date()) current_count = instances.count() - self.assertEqual( - form.submission_count_for_today, current_count) + self.assertEqual(form.submission_count_for_today, current_count) # Confirm that the submission count is decreased # accordingly @@ -2578,8 +2570,7 @@ def test_submission_count_for_day_tracking(self): inst_one.set_deleted() current_count -= 1 - self.assertEqual( - form.submission_count_for_today, current_count) + self.assertEqual(form.submission_count_for_today, current_count) # Confirm submission count isn't decreased if the # date_created is different @@ -2587,9 +2578,7 @@ def test_submission_count_for_day_tracking(self): inst_two.date_created = future_date inst_two.save() inst_two.set_deleted() - self.assertEqual( - form.submission_count_for_today, current_count - ) + self.assertEqual(form.submission_count_for_today, current_count) # Check that deletes made with no current count cached # are successful @@ -2602,22 +2591,22 @@ def test_data_query_multiple_condition(self): data """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) query_str = ( '{"$or":[{"transport/loop_over_transport_types_frequency' '/ambulance/frequency_to_referral_facility":"weekly","t' - 'ransport/loop_over_transport_types_frequency/ambulanc' + "ransport/loop_over_transport_types_frequency/ambulanc" 'e/frequency_to_referral_facility":"daily"}]}' ) - request = self.factory.get(f'/?query={query_str}', **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=self.xform.pk) count = 0 for inst in self.xform.instances.all(): if inst.json.get( - 'transport/loop_over_transport_types_frequency' - '/ambulance/frequency_to_referral_facility' - ) in ['daily', 'weekly']: + "transport/loop_over_transport_types_frequency" + "/ambulance/frequency_to_referral_facility" + ) in ["daily", "weekly"]: count += 1 self.assertEqual(response.status_code, 200) @@ -2626,10 +2615,10 @@ def test_data_query_multiple_condition(self): query_str = ( '{"$or":[{"transport/loop_over_transport_types_frequency' '/ambulance/frequency_to_referral_facility":"weekly"}, {"t' - 'ransport/loop_over_transport_types_frequency/ambulanc' + "ransport/loop_over_transport_types_frequency/ambulanc" 'e/frequency_to_referral_facility":"daily"}]}' ) - request = self.factory.get(f'/?query={query_str}', **self.extra) + request = self.factory.get(f"/?query={query_str}", **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), count) @@ -2640,32 +2629,29 @@ def test_data_query_ornull(self): $or filter option """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - query_str = ('{"$or": [{"_review_status":"3"}' - ', {"_review_status": null }]}') - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = '{"$or": [{"_review_status":"3"}' ', {"_review_status": null }]}' + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) - instances = self.xform.instances.all().order_by('pk') + instances = self.xform.instances.all().order_by("pk") instance = instances[0] self.assertFalse(instance.has_a_review) # Review instance - data = { - "instance": instance.id, - "status": SubmissionReview.APPROVED - } + data = {"instance": instance.id, "status": SubmissionReview.APPROVED} - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -2677,13 +2663,11 @@ def test_data_query_ornull(self): self.assertEqual(len(response.data), 3) # Switch review to pending - data = { - "instance": instance.id, - "status": SubmissionReview.PENDING - } + data = {"instance": instance.id, "status": SubmissionReview.PENDING} - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() @@ -2698,19 +2682,22 @@ def test_data_query_ornull(self): data = { "instance": instance.id, "status": SubmissionReview.REJECTED, - "note": "Testing" + "note": "Testing", } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() # Assert ornull operator still works with multiple values - query_str = ('{"$or": [{"_review_status":"3"},' - ' {"_review_status": "2"}, {"_review_status": null}]}') - request = self.factory.get('/?query=%s' % query_str, **self.extra) + query_str = ( + '{"$or": [{"_review_status":"3"},' + ' {"_review_status": "2"}, {"_review_status": null}]}' + ) + request = self.factory.get("/?query=%s" % query_str, **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 4) @@ -2719,27 +2706,38 @@ def test_data_list_xml_format(self): """Test DataViewSet list XML""" # create submission media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', - 'transport_2011-07-25_19-05-49_2', media_file)) - - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + media_file, + ), + ) + + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk - response = view(request, pk=formid, format='xml') + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) # Ensure response is renderable response.render() - self.assertEqual(response.accepted_media_type, 'application/xml') + self.assertEqual(response.accepted_media_type, "application/xml") instance = self.xform.instances.first() - returned_xml = response.content.decode('utf-8') - server_time = ET.fromstring(returned_xml).attrib.get('serverTime') + returned_xml = response.content.decode("utf-8") + server_time = ET.fromstring(returned_xml).attrib.get("serverTime") edited = instance.last_edited is not None submission_time = instance.date_created.strftime(MONGO_STRFTIME) attachment = instance.attachments.first() @@ -2750,19 +2748,19 @@ def test_data_list_xml_format(self): f' lastModified="{instance.date_modified.isoformat()}" mediaAllReceived="{instance.media_all_received}" mediaCount="{ instance.media_count }" objectID="{instance.id}" reviewComment="" reviewStatus=""' # noqa f' status="{instance.status}" submissionTime="{submission_time}" submittedBy="{instance.user.username}" totalMedia="{ instance.total_media }">' # noqa f'' # noqa - '' - 'none' # noqa - '' # noqa - '' + "" + "none" # noqa + "" # noqa + "" '1335783522563.jpg' - 'uuid:5b2cc313-fc09-437e-8149-fcd32f695d41' # noqa - '' - '' - '' - f'{attachment.id}1335783522563.jpg{instance.xform.id}{ attachment.media_file.name }{ instance.id }image/jpeg/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=small/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=medium' # noqa - '' - '' - '' + "uuid:5b2cc313-fc09-437e-8149-fcd32f695d41" # noqa + "" + "" + "" + f"{attachment.id}1335783522563.jpg{instance.xform.id}{ attachment.media_file.name }{ instance.id }image/jpeg/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=small/api/v1/files/{attachment.id}?filename={ attachment.media_file.name }&suffix=medium" # noqa + "" + "" + "" ) self.assertEqual(expected_xml, returned_xml) @@ -2770,27 +2768,44 @@ def test_invalid_xml_elements_not_present(self): """ Test invalid XML elements such as #text are not present in XML Response """ - self._make_submission(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml')) + self._make_submission( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ) + ) media_file = "1335783522563.jpg" - self._make_submission_w_attachment(os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', 'transport_2011-07-25_19-05-49_2', - 'transport_2011-07-25_19-05-49_2.xml'), - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', - 'transport_2011-07-25_19-05-49_2', media_file)) + self._make_submission_w_attachment( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + "transport_2011-07-25_19-05-49_2.xml", + ), + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49_2", + media_file, + ), + ) - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='xml') + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="xml") self.assertEqual(response.status_code, 200) # Ensure XML is well formed response.render() - returned_xml = response.content.decode('utf-8') + returned_xml = response.content.decode("utf-8") ET.fromstring(returned_xml) @override_settings(STREAM_DATA=True) @@ -2799,30 +2814,27 @@ def test_data_list_xml_format_no_data(self): # create form xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/forms/tutorial/tutorial.xlsx" + "../fixtures/forms/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk query_str = '{"_date_modified":{"$gt":"2020-01-22T11:42:20"}}' - request = self.factory.get('/?query=%s' % query_str, **self.extra) - response = view(request, pk=formid, format='xml') + request = self.factory.get("/?query=%s" % query_str, **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - returned_xml = ( - ''.join([i.decode('utf-8') for i in response.streaming_content]) - - ) + returned_xml = "".join([i.decode("utf-8") for i in response.streaming_content]) - server_time = ET.fromstring(returned_xml).attrib.get('serverTime') + server_time = ET.fromstring(returned_xml).attrib.get("serverTime") expected_xml = ( '\n' f'' - '' + "" ) self.assertEqual(expected_xml, returned_xml) @@ -2833,30 +2845,30 @@ def test_data_list_xml_format_sort(self): responses """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) + view = DataViewSet.as_view({"get": "list"}) formid = self.xform.pk - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - '-date_modified').values_list('id', flat=True)) - request = self.factory.get( - '/?sort=-_date_modified', **self.extra) - response = view(request, pk=formid, format='xml') + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("-date_modified") + .values_list("id", flat=True) + ) + request = self.factory.get("/?sort=-_date_modified", **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - items_in_order = [ - int(data.get('@objectID')) for data in response.data] + items_in_order = [int(data.get("@objectID")) for data in response.data] self.assertEqual(expected_order, items_in_order) # Test `last_edited` field is sorted correctly - expected_order = list(Instance.objects.filter( - xform=self.xform).order_by( - 'last_edited').values_list('id', flat=True)) - request = self.factory.get( - '/?sort=_last_edited', **self.extra) - response = view(request, pk=formid, format='xml') + expected_order = list( + Instance.objects.filter(xform=self.xform) + .order_by("last_edited") + .values_list("id", flat=True) + ) + request = self.factory.get("/?sort=_last_edited", **self.extra) + response = view(request, pk=formid, format="xml") self.assertEqual(response.status_code, 200) - items_in_order = [ - int(data.get('@objectID')) for data in response.data] + items_in_order = [int(data.get("@objectID")) for data in response.data] self.assertEqual(expected_order, items_in_order) @override_settings(SUBMISSION_RETRIEVAL_THRESHOLD=1) @@ -2866,94 +2878,94 @@ def test_data_paginated_past_threshold(self): the requested data surpasses the SUBMISSION_RETRIEVAL_THRESHOLD """ self._make_submissions() - view = DataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) formid = self.xform.pk count = self.xform.instances.all().count() self.assertTrue(count > 1) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) - self.assertIn('Link', response) + self.assertIn("Link", response) self.assertEqual( - response['Link'], + response["Link"], '; rel="next", ' - '; rel="last"') + '; rel="last"', + ) class TestOSM(TestAbstractViewSet): """ Test OSM endpoints in data_viewset. """ + def setUp(self): super(TestOSM, self).setUp() self._login_user_and_profile() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} def test_data_retrieve_instance_osm_format(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - osm_fixtures_dir = os.path.realpath(os.path.join( - os.path.dirname(__file__), '..', 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') - combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') + osm_fixtures_dir = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") + combined_osm_path = os.path.join(osm_fixtures_dir, "combined.osm") self._publish_xls_form_to_project(xlsform_path=xlsform_path) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - files = [open(path, 'rb') for path in paths] - count = Attachment.objects.filter(extension='osm').count() + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + files = [open(path, "rb") for path in paths] + count = Attachment.objects.filter(extension="osm").count() self._make_submission(submission_path, media_file=files) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) formid = self.xform.pk - dataid = self.xform.instances.latest('date_created').pk - request = self.factory.get('/', **self.extra) + dataid = self.xform.instances.latest("date_created").pk + request = self.factory.get("/", **self.extra) # look at the data/[pk]/[dataid].osm endpoint - view = DataViewSet.as_view({'get': 'list'}) - response1 = view(request, pk=formid, format='osm') + view = DataViewSet.as_view({"get": "list"}) + response1 = view(request, pk=formid, format="osm") self.assertEqual(response1.status_code, 200) # look at the data/[pk]/[dataid].osm endpoint - view = DataViewSet.as_view({'get': 'retrieve'}) - response = view(request, pk=formid, dataid=dataid, format='osm') + view = DataViewSet.as_view({"get": "retrieve"}) + response = view(request, pk=formid, dataid=dataid, format="osm") self.assertEqual(response.status_code, 200) - with open(combined_osm_path, encoding='utf-8') as f: + with open(combined_osm_path, encoding="utf-8") as f: osm = f.read() response.render() - self.assertMultiLineEqual(response.content.decode('utf-8').strip(), - osm.strip()) + self.assertMultiLineEqual( + response.content.decode("utf-8").strip(), osm.strip() + ) # look at the data/[pk].osm endpoint - view = DataViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='osm') + view = DataViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="osm") self.assertEqual(response.status_code, 200) response.render() response1.render() self.assertMultiLineEqual( - response1.content.decode('utf-8').strip(), osm.strip()) + response1.content.decode("utf-8").strip(), osm.strip() + ) self.assertMultiLineEqual( - response.content.decode('utf-8').strip(), osm.strip()) + response.content.decode("utf-8").strip(), osm.strip() + ) # filter using value that exists request = self.factory.get( - '/', - data={"query": '{"osm_road": "OSMWay234134797.osm"}'}, - **self.extra) + "/", data={"query": '{"osm_road": "OSMWay234134797.osm"}'}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(len(response.data), 1) # filter using value that doesn't exists request = self.factory.get( - '/', - data={"query": '{"osm_road": "OSMWay123456789.osm"}'}, - **self.extra) + "/", data={"query": '{"osm_road": "OSMWay123456789.osm"}'}, **self.extra + ) response = view(request, pk=formid) self.assertEqual(len(response.data), 0) diff --git a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py index 6d145b0b73..38f7d14837 100644 --- a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py @@ -1,33 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Test /orgs API endpoint implementation. +""" import json from builtins import str as text from django.contrib.auth.models import User from django.core.cache import cache + from mock import patch from rest_framework import status from onadata.apps.api.models.organization_profile import OrganizationProfile -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet -from onadata.apps.api.tools import (get_or_create_organization_owners_team, - add_user_to_organization) -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tools import ( + add_user_to_organization, + get_or_create_organization_owners_team, +) +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.main.models import UserProfile from onadata.libs.permissions import OwnerRole +# pylint: disable=too-many-public-methods class TestOrganizationProfileViewSet(TestAbstractViewSet): + """ + Test /orgs API endpoint implementation. + """ def setUp(self): - super(self.__class__, self).setUp() - self.view = OrganizationProfileViewSet.as_view({ - 'get': 'list', - 'post': 'create', - 'patch': 'partial_update', - }) + super().setUp() + self.view = OrganizationProfileViewSet.as_view( + { + "get": "list", + "post": "create", + "patch": "partial_update", + } + ) def tearDown(self): """ @@ -38,32 +51,32 @@ def tearDown(self): def test_partial_updates(self): self._org_create() - metadata = {u'computer': u'mac'} + metadata = {"computer": "mac"} json_metadata = json.dumps(metadata) - data = {'metadata': json_metadata} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') - profile = OrganizationProfile.objects.get(name='Dennis') + data = {"metadata": json_metadata} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") + profile = OrganizationProfile.objects.get(name="Dennis") self.assertEqual(response.status_code, 200) self.assertEqual(profile.metadata, metadata) def test_partial_updates_invalid(self): self._org_create() - data = {'name': "a" * 31} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') + data = {"name": "a" * 31} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") self.assertEqual(response.status_code, 400) self.assertEqual( - response.data['name'], - [u'Ensure this field has no more than 30 characters.']) + response.data["name"], ["Ensure this field has no more than 30 characters."] + ) def test_orgs_list(self): self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - del self.company_data['metadata'] + del self.company_data["metadata"] self.assertEqual(response.data, [self.company_data]) # inactive organization @@ -76,129 +89,117 @@ def test_orgs_list(self): def test_orgs_list_for_anonymous_user(self): self._org_create() - request = self.factory.get('/') + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def test_orgs_list_for_authenticated_user(self): self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - del self.company_data['metadata'] + del self.company_data["metadata"] self.assertEqual(response.data, [self.company_data]) def test_orgs_list_shared_with_user(self): authenticated_user = self.user user_in_shared_organization, _ = User.objects.get_or_create( - username='the_stalked') + username="the_stalked" + ) UserProfile.objects.get_or_create( - user=user_in_shared_organization, - name=user_in_shared_organization.username + user=user_in_shared_organization, name=user_in_shared_organization.username ) - unshared_organization, _ = User.objects.get_or_create( - username='NotShared') - unshared_organization_profile, _ = OrganizationProfile\ - .objects.get_or_create( - user=unshared_organization, - creator=authenticated_user) + unshared_organization, _ = User.objects.get_or_create(username="NotShared") + unshared_organization_profile, _ = OrganizationProfile.objects.get_or_create( + user=unshared_organization, creator=authenticated_user + ) - add_user_to_organization(unshared_organization_profile, - authenticated_user) + add_user_to_organization(unshared_organization_profile, authenticated_user) - shared_organization, _ = User.objects.get_or_create(username='Shared') - shared_organization_profile, _ = OrganizationProfile\ - .objects.get_or_create( - user=shared_organization, - creator=user_in_shared_organization) + shared_organization, _ = User.objects.get_or_create(username="Shared") + shared_organization_profile, _ = OrganizationProfile.objects.get_or_create( + user=shared_organization, creator=user_in_shared_organization + ) - add_user_to_organization(shared_organization_profile, - authenticated_user) + add_user_to_organization(shared_organization_profile, authenticated_user) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) self.assertTrue(len(response.data), 2) - request = self.factory.get('/', - data={'shared_with': 'the_stalked'}, - **self.extra) + request = self.factory.get( + "/", data={"shared_with": "the_stalked"}, **self.extra + ) response = self.view(request) self.assertEqual(len(response.data), 1) def test_orgs_list_restricted(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'list' - }) + view = OrganizationProfileViewSet.as_view({"get": "list"}) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') + self.assertEqual(self.user.username, "alice") - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, []) def test_orgs_get(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, {'detail': 'Expected URL keyword argument `user`.'}) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + response.data, {"detail": "Expected URL keyword argument `user`."} + ) + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_get_not_creator(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + alice_data = {"username": "alice", "email": "alice@localhost.com"} previous_user = self.user self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') + self.assertEqual(self.user.username, "alice") self.assertNotEqual(previous_user, self.user) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_get_anon(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/') - response = view(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + view = OrganizationProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/") + response = view(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.company_data) - self.assertIn('users', list(response.data)) - for user in response.data['users']: - self.assertEqual(user['role'], 'owner') - self.assertTrue(isinstance(user['user'], text)) + self.assertIn("users", list(response.data)) + for user in response.data["users"]: + self.assertEqual(user["role"], "owner") + self.assertTrue(isinstance(user["user"], text)) def test_orgs_create(self): self._org_create() @@ -206,161 +207,149 @@ def test_orgs_create(self): def test_orgs_create_without_name(self): data = { - 'org': u'denoinc', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "org": "denoinc", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) - self.assertEqual(response.data, {'name': [u'This field is required.']}) + self.assertEqual(response.data, {"name": ["This field is required."]}) def test_org_create_with_anonymous_user(self): data = { - 'name': u'denoinc', - 'org': u'denoinc', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "name": "denoinc", + "org": "denoinc", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json") + "/", data=json.dumps(data), content_type="application/json" + ) response = self.view(request) self.assertEqual(response.status_code, 401) def test_orgs_members_list(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'members' - }) + view = OrganizationProfileViewSet.as_view({"get": "members"}) - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) - self.assertEqual(response.data, [u'denoinc']) + self.assertNotEqual(response.get("Cache-Control"), None) + self.assertEqual(response.data, ["denoinc"]) def test_add_members_to_org_username_required(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - request = self.factory.post('/', data={}, **self.extra) - response = view(request, user='denoinc') + view = OrganizationProfileViewSet.as_view({"post": "members"}) + request = self.factory.post("/", data={}, **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {u'username': [u"This field may not be null."]}) + self.assertEqual(response.data, {"username": ["This field may not be null."]}) def test_add_members_to_org_user_does_not_exist(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) - data = {'username': 'aboy'} + view = OrganizationProfileViewSet.as_view({"post": "members"}) + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {u'username': [u"User 'aboy' does not exist."]}) + self.assertEqual(response.data, {"username": ["User 'aboy' does not exist."]}) def test_add_members_to_org(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy'} + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_user_org_account(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - username = 'second_inc' + username = "second_inc" # Create second org - org_data = {'org': username} + org_data = {"org": username} self._org_create(org_data=org_data) - data = {'username': username} + data = {"username": username} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {'username': [u'Cannot add org account `second_inc` ' - u'as member.']}) + self.assertEqual( + response.data, + {"username": ["Cannot add org account `second_inc` " "as member."]}, + ) def test_member_sees_orgs_added_to(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'list', - 'post': 'members' - }) - - member = 'aboy' - cur_username = self.profile_data['username'] - self.profile_data['username'] = member + view = OrganizationProfileViewSet.as_view({"get": "list", "post": "members"}) + + member = "aboy" + cur_username = self.profile_data["username"] + self.profile_data["username"] = member self._login_user_and_profile() - self.profile_data['username'] = cur_username + self.profile_data["username"] = cur_username self._login_user_and_profile() - data = {'username': member} + data = {"username": member} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) - self.profile_data['username'] = member + self.profile_data["username"] = member self._login_user_and_profile() expected_data = self.company_data - expected_data['users'].append({ - 'first_name': u'Bob', - 'last_name': u'erama', - 'role': 'member', - 'user': member, - 'gravatar': self.user.profile.gravatar, - }) - del expected_data['metadata'] - - request = self.factory.get('/', **self.extra) + expected_data["users"].append( + { + "first_name": "Bob", + "last_name": "erama", + "role": "member", + "user": member, + "gravatar": self.user.profile.gravatar, + } + ) + del expected_data["metadata"] + + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) response_data = dict(response.data[0]) - returned_users = response_data.pop('users') - expected_users = expected_data.pop('users') + returned_users = response_data.pop("users") + expected_users = expected_data.pop("users") self.assertDictEqual(response_data, expected_data) for user in expected_users: self.assertIn(user, returned_users) @@ -368,419 +357,401 @@ def test_member_sees_orgs_added_to(self): def test_role_for_org_non_owner(self): # creating org with member self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" self._create_user_profile() - data = {'username': 'aboy'} - user_role = 'member' + data = {"username": "aboy"} + user_role = "member" request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertIn('users', list(response.data)) - - for user in response.data['users']: - username = user['user'] - role = user['role'] - expected_role = 'owner' if username == 'denoinc' or \ - username == self.user.username else user_role + self.assertIn("users", list(response.data)) + + for user in response.data["users"]: + username = user["user"] + role = user["role"] + expected_role = ( + "owner" + if username == "denoinc" or username == self.user.username + else user_role + ) self.assertEqual(role, expected_role) def test_add_members_to_org_with_anonymous_user(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self._create_user_profile(extra_post_data={'username': 'aboy'}) - data = {'username': 'aboy'} + self._create_user_profile(extra_post_data={"username": "aboy"}) + data = {"username": "aboy"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json") + "/", data=json.dumps(data), content_type="application/json" + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 401) - self.assertNotEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertNotEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_with_non_member_user(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self._create_user_profile(extra_post_data={'username': 'aboy'}) - data = {'username': 'aboy'} + self._create_user_profile(extra_post_data={"username": "aboy"}) + data = {"username": "aboy"} previous_user = self.user - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(extra_post_data=alice_data) - self.assertEqual(self.user.username, 'alice') - self.assertNotEqual(previous_user, self.user) + self.assertEqual(self.user.username, "alice") + self.assertNotEqual(previous_user, self.user) request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 404) - self.assertNotEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertNotEqual(set(response.data), set(["denoinc", "aboy"])) def test_remove_members_from_org(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members' - }) + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"post": "members", "delete": "members"} + ) - self._create_user_profile(extra_post_data={'username': newname}) + self._create_user_profile(extra_post_data={"username": newname}) - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, [u'denoinc']) + self.assertEqual(response.data, ["denoinc"]) - newname = 'aboy2' - self._create_user_profile(extra_post_data={'username': newname}) + newname = "aboy2" + self._create_user_profile(extra_post_data={"username": newname}) - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - request = self.factory.delete( - '/?username={}'.format(newname), **self.extra) + request = self.factory.delete("/?username={}".format(newname), **self.extra) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, [u'denoinc']) + self.assertEqual(response.data, ["denoinc"]) # Removes users from org projects. # Create a project - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) # Create alice - alice = 'alice' - self._create_user_profile(extra_post_data={'username': alice}) - alice_data = {'username': alice, - 'role': 'owner'} + alice = "alice" + self._create_user_profile(extra_post_data={"username": alice}) + alice_data = {"username": alice, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # alice is in project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] self.assertIn(alice, users_in_users) # remove alice from org - request = self.factory.delete( - '/?username={}'.format(alice), **self.extra) + request = self.factory.delete("/?username={}".format(alice), **self.extra) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertNotIn(response.data, [u'alice']) + self.assertNotIn(response.data, ["alice"]) # alice is also removed from project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] self.assertNotIn(alice, users_in_users) def test_orgs_create_with_mixed_case(self): data = { - 'name': u'denoinc', - 'org': u'DenoINC', - 'city': u'Denoville', - 'country': u'US', - 'home_page': u'deno.com', - 'twitter': u'denoinc', - 'description': u'', - 'address': u'', - 'phonenumber': u'', - 'require_auth': False, + "name": "denoinc", + "org": "DenoINC", + "city": "Denoville", + "country": "US", + "home_page": "deno.com", + "twitter": "denoinc", + "description": "", + "address": "", + "phonenumber": "", + "require_auth": False, } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 201) - data['org'] = 'denoinc' + data["org"] = "denoinc" request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertIn("Organization %s already exists." % data['org'], - response.data['org']) + self.assertIn( + "Organization %s already exists." % data["org"], response.data["org"] + ) def test_publish_xls_form_to_organization_project(self): self._org_create() - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) self._publish_xls_form_to_project() self.assertTrue(OwnerRole.user_has_role(self.user, self.xform)) def test_put_change_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(sorted(response.data), sorted([u'denoinc', newname])) + self.assertEqual(sorted(response.data), sorted(["denoinc", newname])) - user_role = 'editor' - data = {'username': newname, 'role': user_role} + user_role = "editor" + data = {"username": newname, "role": user_role} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(sorted(response.data), sorted([u'denoinc', newname])) + self.assertEqual(sorted(response.data), sorted(["denoinc", newname])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertIn('users', list(response.data)) - - for user in response.data['users']: - username = user['user'] - role = user['role'] - expected_role = 'owner' if username == 'denoinc' or \ - username == self.user.username else user_role + self.assertIn("users", list(response.data)) + + for user in response.data["users"]: + username = user["user"] + role = user["role"] + expected_role = ( + "owner" + if username == "denoinc" or username == self.user.username + else user_role + ) self.assertEqual(role, expected_role) def test_put_require_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - data = {'username': newname} + data = {"username": newname} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) def test_put_bad_role(self): self._org_create() - newname = 'aboy' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - self.profile_data['username'] = newname + newname = "aboy" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + self.profile_data["username"] = newname self._create_user_profile() - data = {'username': newname} + data = {"username": newname} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', newname])) + self.assertEqual(set(response.data), set(["denoinc", newname])) - data = {'username': newname, 'role': 42} + data = {"username": newname, "role": 42} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - @patch('onadata.libs.serializers.organization_member_serializer.send_mail') + @patch("onadata.libs.serializers.organization_member_serializer.send_mail") def test_add_members_to_org_email(self, mock_email): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy', - 'email_msg': 'You have been add to denoinc'} + data = {"username": "aboy", "email_msg": "You have been add to denoinc"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) self.assertTrue(mock_email.called) - mock_email.assert_called_with('aboy, You have been added to Dennis' - ' organisation.', - u'You have been add to denoinc', - 'noreply@ona.io', - (u'aboy@org.com',)) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) - - @patch('onadata.libs.serializers.organization_member_serializer.send_mail') + mock_email.assert_called_with( + "aboy, You have been added to Dennis" " organisation.", + "You have been add to denoinc", + "noreply@ona.io", + ("aboy@org.com",), + ) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) + + @patch("onadata.libs.serializers.organization_member_serializer.send_mail") def test_add_members_to_org_email_custom_subj(self, mock_email): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - self.profile_data['username'] = 'aboy' - self.profile_data['email'] = 'aboy@org.com' + self.profile_data["username"] = "aboy" + self.profile_data["email"] = "aboy@org.com" self._create_user_profile() - data = {'username': 'aboy', - 'email_msg': 'You have been add to denoinc', - 'email_subject': 'Your are made'} + data = { + "username": "aboy", + "email_msg": "You have been add to denoinc", + "email_subject": "Your are made", + } request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) self.assertTrue(mock_email.called) - mock_email.assert_called_with('Your are made', - u'You have been add to denoinc', - 'noreply@ona.io', - (u'aboy@org.com',)) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + mock_email.assert_called_with( + "Your are made", + "You have been add to denoinc", + "noreply@ona.io", + ("aboy@org.com",), + ) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) def test_add_members_to_org_with_role(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" self._create_user_profile() - data = {'username': 'aboy', - 'role': 'editor'} + data = {"username": "aboy", "role": "editor"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['users'][2]['user'], 'aboy') - self.assertEqual(response.data['users'][2]['role'], 'editor') + self.assertEqual(response.data["users"][2]["user"], "aboy") + self.assertEqual(response.data["users"][2]["role"], "editor") def test_add_members_to_owner_role(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve', - 'put': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve", "put": "members"} + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': 'aboy', - 'role': 'owner'} + data = {"username": "aboy", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', u'aboy'])) + self.assertEqual(set(response.data), set(["denoinc", "aboy"])) # getting profile - request = self.factory.get('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.get("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['users'][1]['user'], 'aboy') - self.assertEqual(response.data['users'][1]['role'], 'owner') + self.assertEqual(response.data["users"][1]["user"], "aboy") + self.assertEqual(response.data["users"][1]["role"], "owner") owner_team = get_or_create_organization_owners_team(self.organization) self.assertIn(aboy, owner_team.user_set.all()) # test user removed from owner team when role changed - data = {'username': 'aboy', 'role': 'editor'} + data = {"username": "aboy", "role": "editor"} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) owner_team = get_or_create_organization_owners_team(self.organization) @@ -790,40 +761,37 @@ def test_add_members_to_owner_role(self): def test_org_members_added_to_projects(self): # create org self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'get': 'retrieve', - 'put': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"post": "members", "get": "retrieve", "put": "members"} + ) # create aboy - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': 'aboy', - 'role': 'owner'} + data = {"username": "aboy", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # create a proj - project_data = { - 'owner': self.company_data['user'] - } + project_data = {"owner": self.company_data["user"]} self._project_create(project_data) self._publish_xls_form_to_project() # create alice - self.profile_data['username'] = "alice" + self.profile_data["username"] = "alice" alice = self._create_user_profile().user - alice_data = {'username': 'alice', - 'role': 'owner'} + alice_data = {"username": "alice", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) - response = view(request, user='denoinc') + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) # Assert that user added in org is added to teams in proj @@ -833,191 +801,188 @@ def test_org_members_added_to_projects(self): self.assertTrue(OwnerRole.user_has_role(alice, self.xform)) # Org admins are added to owners in project - projectView = ProjectViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + projectView = ProjectViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = projectView(request, pk=self.project.pk) - project_users = response.data.get('users') - users_in_users = [user['user'] for user in project_users] + project_users = response.data.get("users") + users_in_users = [user["user"] for user in project_users] - self.assertIn('bob', users_in_users) - self.assertIn('denoinc', users_in_users) - self.assertIn('aboy', users_in_users) - self.assertIn('alice', users_in_users) + self.assertIn("bob", users_in_users) + self.assertIn("denoinc", users_in_users) + self.assertIn("aboy", users_in_users) + self.assertIn("alice", users_in_users) def test_put_role_user_none_existent(self): self._org_create() - newname = 'i-do-no-exist' - view = OrganizationProfileViewSet.as_view({ - 'get': 'retrieve', - 'post': 'members', - 'put': 'members' - }) - - data = {'username': newname, 'role': 'editor'} + newname = "i-do-no-exist" + view = OrganizationProfileViewSet.as_view( + {"get": "retrieve", "post": "members", "put": "members"} + ) + + data = {"username": newname, "role": "editor"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) def test_update_org_name(self): self._org_create() # update name - data = {'name': "Dennis2"} - request = self.factory.patch('/', data=data, **self.extra) - response = self.view(request, user='denoinc') - self.assertEqual(response.data['name'], "Dennis2") + data = {"name": "Dennis2"} + request = self.factory.patch("/", data=data, **self.extra) + response = self.view(request, user="denoinc") + self.assertEqual(response.data["name"], "Dennis2") self.assertEqual(response.status_code, 200) # check in user profile endpoint - view_user = UserProfileViewSet.as_view({ - 'get': 'retrieve' - }) - request = self.factory.get('/', **self.extra) + view_user = UserProfileViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) - response = view_user(request, user='denoinc') - self.assertNotEqual(response.get('Cache-Control'), None) + response = view_user(request, user="denoinc") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['name'], "Dennis2") + self.assertEqual(response.data["name"], "Dennis2") def test_org_always_has_admin_or_owner(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'put': 'members', - }) - data = {'username': self.user.username, 'role': 'editor'} + view = OrganizationProfileViewSet.as_view( + { + "put": "members", + } + ) + data = {"username": self.user.username, "role": "editor"} request = self.factory.put( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - { - u'non_field_errors': - [u'Organization cannot be without an owner'] - }) + self.assertEqual( + response.data, + {"non_field_errors": ["Organization cannot be without an owner"]}, + ) def test_owner_not_allowed_to_be_removed(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members', - 'get': 'retrieve', - }) + view = OrganizationProfileViewSet.as_view( + { + "post": "members", + "delete": "members", + "get": "retrieve", + } + ) - self.profile_data['username'] = "aboy" + self.profile_data["username"] = "aboy" aboy = self._create_user_profile().user - data = {'username': aboy.username, - 'role': 'owner'} + data = {"username": aboy.username, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', - aboy.username])) + self.assertEqual(set(response.data), set(["denoinc", aboy.username])) - self.profile_data['username'] = "aboy2" + self.profile_data["username"] = "aboy2" aboy2 = self._create_user_profile().user - data = {'username': aboy2.username, - 'role': 'owner'} + data = {"username": aboy2.username, "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 201) - self.assertEqual(set(response.data), set([u'denoinc', - aboy.username, - aboy2.username])) + self.assertEqual( + set(response.data), set(["denoinc", aboy.username, aboy2.username]) + ) - data = {'username': aboy2.username} + data = {"username": aboy2.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) - for user in [u'denoinc', aboy.username]: + for user in ["denoinc", aboy.username]: self.assertIn(user, response.data) # at this point we have bob and aboy as owners - data = {'username': aboy.username} + data = {"username": aboy.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 200) # at this point we only have bob as the owner - data = {'username': self.user.username} + data = {"username": self.user.username} request = self.factory.delete( - '/', json.dumps(data), - content_type="application/json", **self.extra) + "/", json.dumps(data), content_type="application/json", **self.extra + ) - response = view(request, user='denoinc') + response = view(request, user="denoinc") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - { - u'non_field_errors': - [u'Organization cannot be without an owner'] - }) + self.assertEqual( + response.data, + {"non_field_errors": ["Organization cannot be without an owner"]}, + ) def test_orgs_delete(self): self._org_create() self.assertTrue(self.organization.user.is_active) - view = OrganizationProfileViewSet.as_view({ - 'delete': 'destroy' - }) + view = OrganizationProfileViewSet.as_view({"delete": "destroy"}) - request = self.factory.delete('/', **self.extra) - response = view(request, user='denoinc') + request = self.factory.delete("/", **self.extra) + response = view(request, user="denoinc") self.assertEqual(204, response.status_code) - self.assertEqual(0, OrganizationProfile.objects.filter( - user__username='denoinc').count()) - self.assertEqual(0, User.objects.filter(username='denoinc').count()) + self.assertEqual( + 0, OrganizationProfile.objects.filter(user__username="denoinc").count() + ) + self.assertEqual(0, User.objects.filter(username="denoinc").count()) def test_orgs_non_creator_delete(self): self._org_create() - view = OrganizationProfileViewSet.as_view({ - 'delete': 'members', - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view( + {"delete": "members", "post": "members"} + ) - self.profile_data['username'] = "alice" - self.profile_data['email'] = 'alice@localhost.com' + self.profile_data["username"] = "alice" + self.profile_data["email"] = "alice@localhost.com" self._create_user_profile() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} request = self.factory.post( - '/', data=json.dumps(alice_data), - content_type="application/json", **self.extra) + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) - response = view(request, user='denoinc') - expected_results = [u'denoinc', u'alice'] + response = view(request, user="denoinc") + expected_results = ["denoinc", "alice"] self.assertEqual(status.HTTP_201_CREATED, response.status_code) self.assertEqual(expected_results, response.data) self._login_user_and_profile(extra_post_data=alice_data) - request = self.factory.delete('/', data=json.dumps(alice_data), - content_type="application/json", - **self.extra) - response = view(request, user='denoinc') - expected_results = [u'denoinc'] + request = self.factory.delete( + "/", + data=json.dumps(alice_data), + content_type="application/json", + **self.extra + ) + response = view(request, user="denoinc") + expected_results = ["denoinc"] self.assertEqual(expected_results, response.data) def test_creator_in_users(self): @@ -1026,19 +991,19 @@ def test_creator_in_users(self): in the value of the 'users' key within the response from /orgs """ self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) expected_user = { - 'user': self.user.username, - 'role': 'owner', - 'first_name': 'Bob', - 'last_name': 'erama', - 'gravatar': self.user.profile.gravatar + "user": self.user.username, + "role": "owner", + "first_name": "Bob", + "last_name": "erama", + "gravatar": self.user.profile.gravatar, } - self.assertIn(expected_user, response.data[0]['users']) + self.assertIn(expected_user, response.data[0]["users"]) def test_creator_permissions(self): """ @@ -1046,9 +1011,9 @@ def test_creator_permissions(self): permissions """ self._org_create() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) orgs = OrganizationProfile.objects.filter(creator=self.user) @@ -1056,41 +1021,36 @@ def test_creator_permissions(self): org = orgs.first() self.assertTrue(OwnerRole.user_has_role(self.user, org)) - self.assertTrue( - OwnerRole.user_has_role(self.user, org.userprofile_ptr)) + self.assertTrue(OwnerRole.user_has_role(self.user, org.userprofile_ptr)) - members_view = OrganizationProfileViewSet.as_view({ - 'post': 'members', - 'delete': 'members' - }) + members_view = OrganizationProfileViewSet.as_view( + {"post": "members", "delete": "members"} + ) # New admins should also have the required permissions - self.profile_data['username'] = "dave" + self.profile_data["username"] = "dave" dave = self._create_user_profile().user - data = {'username': 'dave', - 'role': 'owner'} + data = {"username": "dave", "role": "owner"} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) - response = members_view(request, user='denoinc') + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = members_view(request, user="denoinc") self.assertEqual(response.status_code, 201) # Ensure user has role self.assertTrue(OwnerRole.user_has_role(dave, org)) - self.assertTrue( - OwnerRole.user_has_role(dave, org.userprofile_ptr)) + self.assertTrue(OwnerRole.user_has_role(dave, org.userprofile_ptr)) # Permissions should be removed when the user is removed from # organization - request = self.factory.delete('/', data=json.dumps(data), - content_type="application/json", - **self.extra) - response = members_view(request, user='denoinc') - expected_results = [u'denoinc'] + request = self.factory.delete( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) + response = members_view(request, user="denoinc") + expected_results = ["denoinc"] self.assertEqual(expected_results, response.data) # Ensure permissions are removed self.assertFalse(OwnerRole.user_has_role(dave, org)) - self.assertFalse( - OwnerRole.user_has_role(dave, org.userprofile_ptr)) + self.assertFalse(OwnerRole.user_has_role(dave, org.userprofile_ptr)) diff --git a/onadata/apps/api/tests/viewsets/test_widget_viewset.py b/onadata/apps/api/tests/viewsets/test_widget_viewset.py index 0a7da9931d..93ec7cf33a 100644 --- a/onadata/apps/api/tests/viewsets/test_widget_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_widget_viewset.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test /widgets API endpoint implementation. +""" import os import json @@ -5,51 +9,65 @@ from django.contrib.contenttypes.models import ContentType from onadata.apps.logger.models.widget import Widget -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.widget_viewset import WidgetViewSet from onadata.libs.permissions import ReadOnlyRole from onadata.libs.permissions import DataEntryOnlyRole from onadata.libs.permissions import OwnerRole from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.viewsets.organization_profile_viewset import\ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) +# pylint: disable=too-many-public-methods class TestWidgetViewSet(TestAbstractViewSet): + """ + Test /widgets API endpoint implementation. + """ + def setUp(self): super(self.__class__, self).setUp() xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._org_create() self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self._create_dataview() - self.view = WidgetViewSet.as_view({ - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update', - 'delete': 'destroy', - 'get': 'retrieve', - }) + self.view = WidgetViewSet.as_view( + { + "post": "create", + "put": "update", + "patch": "partial_update", + "delete": "destroy", + "get": "retrieve", + } + ) def test_create_widget(self): self._create_widget() def test_create_only_mandatory_fields(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) @@ -57,11 +75,11 @@ def test_create_only_mandatory_fields(self): def test_create_using_dataview(self): data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) @@ -69,64 +87,61 @@ def test_create_using_dataview(self): def test_create_using_unsupported_model_source(self): data = { - 'content_object': 'http://testserver/api/v1/projects/%s' % - self.project.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/projects/%s" % self.project.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual(count, Widget.objects.all().count()) self.assertEqual( - response.data['content_object'], - [u"`%s` is not a valid relation." % data['content_object']] + response.data["content_object"], + ["`%s` is not a valid relation." % data["content_object"]], ) def test_create_without_required_field(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual(count, Widget.objects.all().count()) - self.assertEqual(response.data['column'], - [u"This field is required."]) + self.assertEqual(response.data["column"], ["This field is required."]) def test_create_unsupported_widget_type(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "table", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "table", + "view_type": "horizontal-bar", + "column": "_submission_time", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual(count, Widget.objects.all().count()) - self.assertEqual(response.data['widget_type'], - [u'"%s" is not a valid choice.' - % data['widget_type']]) + self.assertEqual( + response.data["widget_type"], + ['"%s" is not a valid choice.' % data["widget_type"]], + ) def test_update_widget(self): self._create_widget() @@ -134,56 +149,53 @@ def test_update_widget(self): key = self.widget.key data = { - 'title': 'My new title updated', - 'description': 'new description', - 'aggregation': 'new aggregation', - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "My new title updated", + "description": "new description", + "aggregation": "new aggregation", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) - self.widget = Widget.objects.all().order_by('pk').reverse()[0] + self.widget = Widget.objects.all().order_by("pk").reverse()[0] self.assertEqual(key, self.widget.key) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['title'], 'My new title updated') - self.assertEqual(response.data['key'], key) - self.assertEqual(response.data['description'], - "new description") - self.assertEqual(response.data['aggregation'], - "new aggregation") + self.assertEqual(response.data["title"], "My new title updated") + self.assertEqual(response.data["key"], key) + self.assertEqual(response.data["description"], "new description") + self.assertEqual(response.data["aggregation"], "new aggregation") def test_patch_widget(self): self._create_widget() data = { - 'column': "_submitted_by", + "column": "_submitted_by", } - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['column'], '_submitted_by') + self.assertEqual(response.data["column"], "_submitted_by") def test_delete_widget(self): - ct = ContentType.objects.get(model='xform', app_label='logger') + ct = ContentType.objects.get(model="xform", app_label="logger") self._create_widget() - count = Widget.objects.filter(content_type=ct, - object_id=self.xform.pk).count() + count = Widget.objects.filter(content_type=ct, object_id=self.xform.pk).count() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 204) - after_count = Widget.objects.filter(content_type=ct, - object_id=self.xform.pk).count() + after_count = Widget.objects.filter( + content_type=ct, object_id=self.xform.pk + ).count() self.assertEqual(count - 1, after_count) def test_list_widgets(self): @@ -191,20 +203,21 @@ def test_list_widgets(self): self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -212,53 +225,50 @@ def test_list_widgets(self): def test_widget_permission_create(self): - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = WidgetViewSet.as_view({ - 'post': 'create' - }) + view = WidgetViewSet.as_view({"post": "create"}) data = { - 'title': "Widget that", - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'description': "Test widget", - 'aggregation': "Sum", - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "age", - 'group_by': '' + "title": "Widget that", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "description": "Test widget", + "aggregation": "Sum", + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "age", + "group_by": "", } # to do: test random user with auth but no perms - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) self.assertEqual(response.status_code, 400) # owner OwnerRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) self.assertEqual(response.status_code, 201) # readonly ReadOnlyRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) self.assertEqual(response.status_code, 201) # dataentryonlyrole DataEntryOnlyRole.add(self.user, self.project) - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **self.extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request) self.assertEqual(response.status_code, 201) @@ -266,41 +276,43 @@ def test_widget_permission_create(self): def test_widget_permission_change(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) data = { - 'title': "Widget those", + "title": "Widget those", } OwnerRole.add(self.user, self.project) OwnerRole.add(self.user, self.xform) - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['title'], 'Widget those') + self.assertEqual(response.data["title"], "Widget those") ReadOnlyRole.add(self.user, self.project) ReadOnlyRole.add(self.user, self.xform) - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['title'], 'Widget those') + self.assertEqual(response.data["title"], "Widget those") def test_widget_permission_list(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -309,7 +321,7 @@ def test_widget_permission_list(self): # assign alice the perms ReadOnlyRole.add(self.user, self.xform) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -318,10 +330,10 @@ def test_widget_permission_list(self): def test_widget_permission_get(self): self._create_widget() - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 404) @@ -329,142 +341,137 @@ def test_widget_permission_get(self): # assign alice the perms ReadOnlyRole.add(self.user, self.project) - request = self.factory.get('/', **self.extra) - response = self.view(request, formid=self.xform.pk, - pk=self.widget.pk) + request = self.factory.get("/", **self.extra) + response = self.view(request, formid=self.xform.pk, pk=self.widget.pk) self.assertEqual(response.status_code, 200) def test_widget_data(self): self._create_widget() - data = { - "data": True - } + data = {"data": True} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEqual(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) - - self.assertEqual(response.data.get('data')['data_type'], 'numeric') - self.assertEqual(response.data.get('data')['field_label'], - 'How old are you?') - self.assertEqual(response.data.get('data')['field_type'], 'integer') - self.assertEqual(response.data.get('data')['field_xpath'], 'age') + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) + + self.assertEqual(response.data.get("data")["data_type"], "numeric") + self.assertEqual(response.data.get("data")["field_label"], "How old are you?") + self.assertEqual(response.data.get("data")["field_type"], "integer") + self.assertEqual(response.data.get("data")["field_xpath"], "age") def test_widget_data_with_group_by(self): - self._create_widget(group_by='gender') + self._create_widget(group_by="gender") - data = { - "data": True - } + data = {"data": True} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEqual(len(response.data.get('data')['data']), 2) - self.assertIn('gender', response.data.get('data')['data'][0]) - self.assertIn('sum', response.data.get('data')['data'][0]) - self.assertIn('mean', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 2) + self.assertIn("gender", response.data.get("data")["data"][0]) + self.assertIn("sum", response.data.get("data")["data"][0]) + self.assertIn("mean", response.data.get("data")["data"][0]) def test_widget_data_widget(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "gender", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "gender", } self._create_widget(data) - data = { - "data": True - } - request = self.factory.get('/', data=data, **self.extra) + data = {"data": True} + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertEqual(response.data.get('data'), - { - 'field_type': u'select one', - 'data_type': 'categorized', - 'field_xpath': u'gender', - 'field_label': u'Gender', - 'grouped_by': None, - 'data': [ - {'count': 1, 'gender': u'female'}, - {'count': 7, 'gender': u'male'}]}) + self.assertIsNotNone(response.data.get("data")) + self.assertEqual( + response.data.get("data"), + { + "field_type": "select one", + "data_type": "categorized", + "field_xpath": "gender", + "field_label": "Gender", + "grouped_by": None, + "data": [ + {"count": 1, "gender": "female"}, + {"count": 7, "gender": "male"}, + ], + }, + ) def test_widget_with_key(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": self.widget.key - } + data = {"key": self.widget.key} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEqual(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) def test_widget_with_key_anon(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": self.widget.key - } + data = {"key": self.widget.key} # Anonymous user can access the widget self.extra = {} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertIn('data', response.data.get('data')) - self.assertEqual(len(response.data.get('data')['data']), 8) - self.assertIn('age', response.data.get('data')['data'][0]) - self.assertIn('count', response.data.get('data')['data'][0]) + self.assertIsNotNone(response.data.get("data")) + self.assertIn("data", response.data.get("data")) + self.assertEqual(len(response.data.get("data")["data"]), 8) + self.assertIn("age", response.data.get("data")["data"][0]) + self.assertIn("count", response.data.get("data")["data"][0]) def test_widget_with_nonexistance_key(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "key": "randomkeythatdoesnotexist" - } + data = {"key": "randomkeythatdoesnotexist"} self.extra = {} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request, pk=self.xform.pk) self.assertEqual(response.status_code, 404) @@ -472,12 +479,14 @@ def test_widget_with_nonexistance_key(self): def test_widget_data_public_form(self): self._create_widget() - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) self.extra = {} - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -487,7 +496,7 @@ def test_widget_data_public_form(self): self.xform.shared_data = True self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, formid=self.xform.pk) self.assertEqual(response.status_code, 200) @@ -497,46 +506,42 @@ def test_widget_pk_formid_required(self): self._create_widget() data = { - 'title': 'My new title updated', - 'description': 'new description', - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "My new title updated", + "description": "new description", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.put('/', data=data, **self.extra) + request = self.factory.put("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {u'detail': u"'pk' required for this" - u" action"}) + self.assertEqual(response.data, {"detail": "'pk' required for this" " action"}) def test_list_widgets_with_formid(self): self._create_widget() self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "xform": self.xform.pk - } + data = {"xform": self.xform.pk} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -544,174 +549,177 @@ def test_list_widgets_with_formid(self): def test_create_column_not_in_form(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "doesnotexists", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "doesnotexists", } count = Widget.objects.all().count() - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) self.assertEqual(count, Widget.objects.all().count()) - self.assertEqual(response.data['column'], - [u"'doesnotexists' not in the form."]) + self.assertEqual(response.data["column"], ["'doesnotexists' not in the form."]) def test_create_widget_with_xform_no_perms(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "age", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "age", } - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._login_user_and_profile(alice_data) - request = self.factory.post('/', data=data, **self.extra) + request = self.factory.post("/", data=data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['content_object'], - [u"You don't have permission to the Project."]) + self.assertEqual( + response.data["content_object"], + ["You don't have permission to the Project."], + ) def test_filter_widgets_by_dataview(self): self._create_widget() self._publish_xls_form_to_project() data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submitted_by", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submitted_by", } self._create_widget(data=data) data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) - view = WidgetViewSet.as_view({ - 'get': 'list', - }) + view = WidgetViewSet.as_view( + { + "get": "list", + } + ) - data = { - "dataview": self.data_view.pk - } + data = {"dataview": self.data_view.pk} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 2) - data = { - "dataview": "so_invalid" - } + data = {"dataview": "so_invalid"} - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data['detail'], - u"Invalid value for dataview %s." % "so_invalid") + self.assertEqual( + response.data["detail"], "Invalid value for dataview %s." % "so_invalid" + ) def test_order_widget(self): self._create_widget() self._create_widget() self._create_widget() - data = { - 'column': "_submission_time", - 'order': 1 - } + data = {"column": "_submission_time", "order": 1} - request = self.factory.patch('/', data=data, **self.extra) + request = self.factory.patch("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['order'], 1) + self.assertEqual(response.data["order"], 1) - widget = Widget.objects.all().order_by('pk')[0] + widget = Widget.objects.all().order_by("pk")[0] self.assertEqual(widget.order, 0) - widget = Widget.objects.all().order_by('pk')[1] + widget = Widget.objects.all().order_by("pk")[1] self.assertEqual(widget.order, 2) def test_widget_data_case_sensitive(self): xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial_2.xlsx") + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial_2.xlsx", + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial_2', 'instances', 'uuid{}'.format(x), - 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial_2", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "Gender", - 'metadata': { - "test metadata": "percentage" - } + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "Gender", + "metadata": {"test metadata": "percentage"}, } self._create_widget(data) - data = { - "data": True - } - request = self.factory.get('/', data=data, **self.extra) + data = {"data": True} + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=self.widget.pk) self.assertEqual(response.status_code, 200) - self.assertIsNotNone(response.data.get('data')) - self.assertEqual(response.data.get('data'), - { - 'field_type': u'select one', - 'data_type': 'categorized', - 'field_xpath': u'Gender', - 'field_label': u'Gender', - 'grouped_by': None, - 'data': [ - {'count': 1, 'Gender': u'female'}, - {'count': 7, 'Gender': u'male'}]}) + self.assertIsNotNone(response.data.get("data")) + self.assertEqual( + response.data.get("data"), + { + "field_type": "select one", + "data_type": "categorized", + "field_xpath": "Gender", + "field_label": "Gender", + "grouped_by": None, + "data": [ + {"count": 1, "Gender": "female"}, + {"count": 7, "Gender": "male"}, + ], + }, + ) def test_widget_create_by_org_admin(self): self.project.organization = self.organization.user self.project.save() - chuck_data = {'username': 'chuck', 'email': 'chuck@localhost.com'} + chuck_data = {"username": "chuck", "email": "chuck@localhost.com"} chuck_profile = self._create_user_profile(chuck_data) - view = OrganizationProfileViewSet.as_view({ - 'post': 'members' - }) + view = OrganizationProfileViewSet.as_view({"post": "members"}) - data = {'username': chuck_profile.user.username, - 'role': OwnerRole.name} + data = {"username": chuck_profile.user.username, "role": OwnerRole.name} request = self.factory.post( - '/', data=json.dumps(data), - content_type="application/json", **self.extra) + "/", data=json.dumps(data), content_type="application/json", **self.extra + ) response = view(request, user=self.organization.user.username) @@ -720,88 +728,78 @@ def test_widget_create_by_org_admin(self): owners_team = get_or_create_organization_owners_team(self.organization) self.assertIn(chuck_profile.user, owners_team.user_set.all()) - extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % chuck_profile.user.auth_token} + extra = {"HTTP_AUTHORIZATION": "Token %s" % chuck_profile.user.auth_token} - view = WidgetViewSet.as_view({ - 'post': 'create' - }) + view = WidgetViewSet.as_view({"post": "create"}) data = { - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } - request = self.factory.post('/', data=json.dumps(data), - content_type="application/json", - **extra) + request = self.factory.post( + "/", data=json.dumps(data), content_type="application/json", **extra + ) response = view(request) self.assertEqual(response.status_code, 201) def test_create_multiple_choice(self): data = { - 'content_object': 'http://testserver/api/v1/forms/%s' % - self.xform.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "favorite_toppings/pepperoni", + "content_object": "http://testserver/api/v1/forms/%s" % self.xform.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "favorite_toppings/pepperoni", } self._create_widget(data) - data = { - "data": True - } + data = {"data": True} pk = self.widget.pk - request = self.factory.get('/', data=data, **self.extra) + request = self.factory.get("/", data=data, **self.extra) response = self.view(request, pk=pk) self.assertEqual(response.status_code, 200) form_pk = self.xform.pk expected = { - 'content_object': - u'http://testserver/api/v1/forms/{}'.format(form_pk), - 'description': None, - 'title': None, - 'url': u'http://testserver/api/v1/widgets/{}'.format(pk), - 'view_type': u'horizontal-bar', - 'aggregation': None, - 'order': 0, - 'widget_type': 'charts', - 'column': u'favorite_toppings/pepperoni', - 'group_by': None, - 'key': self.widget.key, - 'data': { - 'field_type': u'', - 'data_type': 'categorized', - 'field_xpath': u'favorite_toppings/pepperoni', - 'grouped_by': None, - 'field_label': u'Pepperoni', - 'data': [{'' - 'count': 0, - 'favorite_toppings/pepperoni': None - }] - }, - 'id': pk, - 'metadata': {} + "content_object": "http://testserver/api/v1/forms/{}".format(form_pk), + "description": None, + "title": None, + "url": "http://testserver/api/v1/widgets/{}".format(pk), + "view_type": "horizontal-bar", + "aggregation": None, + "order": 0, + "widget_type": "charts", + "column": "favorite_toppings/pepperoni", + "group_by": None, + "key": self.widget.key, + "data": { + "field_type": "", + "data_type": "categorized", + "field_xpath": "favorite_toppings/pepperoni", + "grouped_by": None, + "field_label": "Pepperoni", + "data": [{"" "count": 0, "favorite_toppings/pepperoni": None}], + }, + "id": pk, + "metadata": {}, } self.assertEqual(expected, response.data) def test_create_long_title(self): data = { - 'title': 'When editing grouped charts titles, much as the title ' - 'can be edited, it cant be saved as the title exceeds 50', - 'content_object': 'http://testserver/api/v1/dataviews/%s' % - self.data_view.pk, - 'widget_type': "charts", - 'view_type': "horizontal-bar", - 'column': "_submission_time", + "title": "When editing grouped charts titles, much as the title " + "can be edited, it cant be saved as the title exceeds 50", + "content_object": "http://testserver/api/v1/dataviews/%s" + % self.data_view.pk, + "widget_type": "charts", + "view_type": "horizontal-bar", + "column": "_submission_time", } self._create_widget(data) diff --git a/onadata/apps/logger/tests/test_form_submission.py b/onadata/apps/logger/tests/test_form_submission.py index b18eb69bcb..52a7ee33fa 100644 --- a/onadata/apps/logger/tests/test_form_submission.py +++ b/onadata/apps/logger/tests/test_form_submission.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test data submissions. +""" import os import re @@ -31,6 +35,7 @@ def catch_signal(signal): signal.disconnect(handler) +# pylint: disable=too-many-public-methods class TestFormSubmission(TestBase): """ Testing POSTs to "/submission" @@ -40,7 +45,7 @@ def setUp(self): TestBase.setUp(self) xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/tutorial.xlsx" + "../fixtures/tutorial/tutorial.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) @@ -50,7 +55,7 @@ def test_form_post(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) @@ -60,9 +65,9 @@ def test_duplicate_form_id(self): """ Should return an error if submitting to a form with a duplicate ID. """ - project = Project.objects.create(name="another project", - organization=self.user, - created_by=self.user) + project = Project.objects.create( + name="another project", organization=self.user, created_by=self.user + ) first_xform = XForm.objects.first() first_xform.pk = None first_xform.project = project @@ -70,41 +75,40 @@ def test_duplicate_form_id(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) self.assertTrue( "Unable to submit because there are multiple forms with this form" - in self.response.content.decode('utf-8')) + in self.response.content.decode("utf-8") + ) - @patch('django.utils.datastructures.MultiValueDict.pop') + @patch("django.utils.datastructures.MultiValueDict.pop") def test_fail_with_ioerror_read(self, mock_pop): - mock_pop.side_effect = IOError( - 'request data read error') + mock_pop.side_effect = IOError("request data read error") self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) self.assertEqual(0, self.xform.instances.count()) - @patch('django.utils.datastructures.MultiValueDict.pop') + @patch("django.utils.datastructures.MultiValueDict.pop") def test_fail_with_ioerror_wsgi(self, mock_pop): - mock_pop.side_effect = IOError( - 'error during read(65536) on wsgi.input') + mock_pop.side_effect = IOError("error during read(65536) on wsgi.input") self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) @@ -122,15 +126,16 @@ def test_submission_to_require_auth_anon(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) # create a new user - username = 'alice' + username = "alice" self._create_user(username, username) - self._make_submission(xml_submission_file_path, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, auth=DigestAuth("alice", "alice") + ) self.assertEqual(self.response.status_code, 403) def test_submission_to_require_auth_without_perm(self): @@ -144,15 +149,16 @@ def test_submission_to_require_auth_without_perm(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) # create a new user - username = 'alice' + username = "alice" self._create_user(username, username) - self._make_submission(xml_submission_file_path, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, auth=DigestAuth("alice", "alice") + ) self.assertEqual(self.response.status_code, 403) @@ -172,16 +178,16 @@ def test_submission_to_require_auth_with_perm(self): self.assertTrue(self.xform.require_auth) # create a new user - username = 'alice' + username = "alice" alice = self._create_user(username, username) # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) auth = DigestAuth(username, username) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) @@ -190,7 +196,7 @@ def test_form_post_to_missing_form(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial/instances/" - "tutorial_invalid_id_string_2012-06-27_11-27-53.xml" + "tutorial_invalid_id_string_2012-06-27_11-27-53.xml", ) with self.assertRaises(Http404): self._make_submission(path=xml_submission_file_path) @@ -201,13 +207,13 @@ def test_duplicate_submissions(self): """ xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/test_forms/survey_names/survey_names.xlsx" + "../fixtures/test_forms/survey_names/survey_names.xlsx", ) self._publish_xls_file(xls_file_path) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/test_forms/survey_names/instances/" - "survey_names_2012-08-17_11-24-53.xml" + "survey_names_2012-08-17_11-24-53.xml", ) self._make_submission(xml_submission_file_path) @@ -216,34 +222,38 @@ def test_duplicate_submissions(self): self.assertEqual(self.response.status_code, 202) def test_unicode_submission(self): - """Test xml submissions that contain unicode characters - """ + """Test xml submissions that contain unicode characters""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_unicode_submission.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_unicode_submission.xml", ) self.user.profile.require_auth = True self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) client = DigestClient() - client.set_authorization('alice', 'alice', 'Digest') + client.set_authorization("alice", "alice", "Digest") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) def test_duplicate_submission_with_same_instanceid(self): - """Test duplicate xml submissions - """ + """Test duplicate xml submissions""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -252,17 +262,22 @@ def test_duplicate_submission_with_same_instanceid(self): self.assertEqual(self.response.status_code, 202) def test_duplicate_submission_with_different_content(self): - """Test xml submissions with same instancID but different content - """ + """Test xml submissions with same instancID but different content""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) duplicate_xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_same_instanceID.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_same_instanceID.xml", ) pre_count = Instance.objects.count() @@ -289,20 +304,18 @@ def test_edited_submission(self): xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} cursor = [r for r in query_data(**query_args)] - num_data_instances = cursor[0]['count'] + num_data_instances = cursor[0]["count"] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -315,82 +328,86 @@ def test_edited_submission(self): self.assertIsNone(initial_instance.json.get(LAST_EDITED)) # no new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # edited submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) client = DigestClient() - client.set_authorization('bob', 'bob', 'Digest') + client.set_authorization("bob", "bob", "Digest") with catch_signal(process_submission) as handler: self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 1) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) instance_history_1 = InstanceHistory.objects.first() edited_instance = self.xform.instances.first() - self.assertDictEqual(initial_instance.get_dict(), - instance_history_1.get_dict()) - handler.assert_called_once_with(instance=edited_instance, - sender=Instance, signal=ANY) + self.assertDictEqual(initial_instance.get_dict(), instance_history_1.get_dict()) + handler.assert_called_once_with( + instance=edited_instance, sender=Instance, signal=ANY + ) self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid) # check that instance history's submission_date is equal to instance's # date_created - last_edited by default is null for an instance - self.assertEqual(edited_instance.date_created, - instance_history_1.submission_date) + self.assertEqual( + edited_instance.date_created, instance_history_1.submission_date + ) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row - query_args['count'] = False + query_args["count"] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_edit_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?(.+?)", xml_str).groups()[0] - self.assertEqual(record['name'], edited_name) + self.assertEqual(record["name"], edited_name) instance_before_second_edit = edited_instance xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml", ) self._make_submission(xml_edit_submission_file_path) cursor = query_data(**query_args) record = cursor[0] edited_instance = self.xform.instances.first() instance_history_2 = InstanceHistory.objects.last() - self.assertEqual(instance_before_second_edit.last_edited, - instance_history_2.submission_date) + self.assertEqual( + instance_before_second_edit.last_edited, instance_history_2.submission_date + ) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) - self.assertEqual(record['name'], 'Tom and Jerry') - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 2) + self.assertEqual(record["name"], "Tom and Jerry") + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2) # submitting original submission is treated as a duplicate # does not add a new record # does not change data self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 202) self.assertEqual(Instance.objects.count(), num_instances + 1) - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 2) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2) def test_submission_w_mismatched_uuid(self): """ @@ -400,8 +417,11 @@ def test_submission_w_mismatched_uuid(self): # submit instance with uuid that would not match the forms xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml", ) self._make_submission(xml_submission_file_path) @@ -414,47 +434,53 @@ def _test_fail_submission_if_no_username(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_xform_uuid.xml", ) with self.assertRaises(Http404): - self._make_submission(path=xml_submission_file_path, username='') + self._make_submission(path=xml_submission_file_path, username="") def test_fail_submission_if_bad_id_string(self): - """Test that a submission fails if the uuid's don't match. - """ + """Test that a submission fails if the uuid's don't match.""" xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_bad_id_string.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_bad_id_string.xml", ) with self.assertRaises(Http404): self._make_submission(path=xml_submission_file_path) def test_edit_updated_geopoint_cache(self): - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field - query_args['count'] = False + query_args["count"] = False records = query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -462,7 +488,7 @@ def test_edit_updated_geopoint_cache(self): self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field - gps = records[0]['gps'].split(" ") + gps = records[0]["gps"].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1])) @@ -471,18 +497,17 @@ def test_submission_when_requires_auth(self): self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) - auth = DigestAuth('alice', 'alice') - self._make_submission( - xml_submission_file_path, auth=auth) + auth = DigestAuth("alice", "alice") + self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) def test_submission_linked_to_reporter(self): @@ -490,19 +515,18 @@ def test_submission_linked_to_reporter(self): self.user.profile.save() # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.create(user=alice) # assign report perms to user - assign_perm('report_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) - auth = DigestAuth('alice', 'alice') - self._make_submission( - xml_submission_file_path, auth=auth) + auth = DigestAuth("alice", "alice") + self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) instance = Instance.objects.all().reverse()[0] self.assertEqual(instance.user, alice) @@ -513,8 +537,11 @@ def test_edited_submission_require_auth(self): """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) # require authentication self.user.profile.require_auth = True @@ -522,75 +549,71 @@ def test_edited_submission_require_auth(self): num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() - query_args = { - 'xform': self.xform, - 'query': '{}', - 'fields': '[]', - 'count': True - } + query_args = {"xform": self.xform, "query": "{}", "fields": "[]", "count": True} cursor = query_data(**query_args) - num_data_instances = cursor[0]['count'] + num_data_instances = cursor[0]["count"] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # create a new user - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.create(user=alice) - auth = DigestAuth('alice', 'alice') + auth = DigestAuth("alice", "alice") # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "..", "fixtures", "tutorial", "instances", - "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user - assign_perm('report_xform', alice, self.xform) - assign_perm('logger.change_xform', alice, self.xform) + assign_perm("report_xform", alice, self.xform) + assign_perm("logger.change_xform", alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history - self.assertEqual( - InstanceHistory.objects.count(), num_instances_history + 1) + self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = query_data(**query_args) - self.assertEqual(cursor[0]['count'], num_data_instances + 1) + self.assertEqual(cursor[0]["count"], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row - query_args['count'] = False + query_args["count"] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?(.+?)", xml_str).groups()[0] - self.assertEqual(record['name'], edited_name) + self.assertEqual(record["name"], edited_name) - @patch('onadata.libs.utils.logger_tools.create_instance') + @patch("onadata.libs.utils.logger_tools.create_instance") def test_fail_with_unreadable_post_error(self, mock_create_instance): """Test UnreadablePostError is handled on form data submission""" mock_create_instance.side_effect = UnreadablePostError( - 'error during read(65536) on wsgi.input' + "error during read(65536) on wsgi.input" ) self.assertEqual(0, self.xform.instances.count()) xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml" + "../fixtures/tutorial/instances/tutorial_2012-06-27_11-27-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 400) @@ -607,15 +630,15 @@ def test_form_submission_with_infinity_values(self): This test confirms that we are handling such cases and they do not result in 500 response codes. """ - xls_file_path = os.path.join(os.path.dirname( - os.path.abspath(__file__)), "../tests/fixtures/infinity.xlsx" + xls_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../tests/fixtures/infinity.xlsx", ) self._publish_xls_file_and_set_xform(xls_file_path) - xml_submission_file_path = os.path.join(os.path.dirname( - os.path.abspath(__file__)), "../tests/fixtures/infinity.xml" + xml_submission_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../tests/fixtures/infinity.xml" ) self._make_submission(path=xml_submission_file_path) self.assertEqual(400, self.response.status_code) - self.assertIn( - 'invalid input syntax for type json', str(self.response.message)) + self.assertIn("invalid input syntax for type json", str(self.response.message)) diff --git a/onadata/apps/main/tests/test_form_metadata.py b/onadata/apps/main/tests/test_form_metadata.py index c7012e1a62..a0e72c1e11 100644 --- a/onadata/apps/main/tests/test_form_metadata.py +++ b/onadata/apps/main/tests/test_form_metadata.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test form MetaData objects. +""" import hashlib import os from builtins import open @@ -10,112 +14,147 @@ from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.main.views import show, edit, download_metadata, \ - download_media_data, delete_metadata +from onadata.apps.main.views import ( + show, + edit, + download_metadata, + download_media_data, + delete_metadata, +) from onadata.libs.utils.cache_tools import XFORM_METADATA_CACHE +# pylint: disable=too-many-public-methods class TestFormMetadata(TestBase): + """ + Test form MetaData objects. + """ def setUp(self): TestBase.setUp(self) self._create_user_and_login() self._publish_transportation_form_and_submit_instance() - self.url = reverse(show, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) - self.edit_url = reverse(edit, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) - - def _add_metadata(self, data_type='doc'): - if data_type == 'media': - name = 'screenshot.png' + self.url = reverse( + show, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + self.edit_url = reverse( + edit, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + + def _add_metadata(self, data_type="doc"): + if data_type == "media": + name = "screenshot.png" else: - name = 'transportation.xlsx' - path = os.path.join(self.this_directory, "fixtures", - "transportation", name) - with open(path, 'rb') as doc_file: + name = "transportation.xlsx" + path = os.path.join(self.this_directory, "fixtures", "transportation", name) + with open(path, "rb") as doc_file: self.post_data = {} self.post_data[data_type] = doc_file self.client.post(self.edit_url, self.post_data) - if data_type == 'media': - self.doc = MetaData.objects.filter(data_type='media').reverse()[0] - self.doc_url = reverse(download_media_data, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': self.doc.id}) + if data_type == "media": + self.doc = MetaData.objects.filter(data_type="media").reverse()[0] + self.doc_url = reverse( + download_media_data, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": self.doc.id, + }, + ) else: self.doc = MetaData.objects.all().reverse()[0] - self.doc_url = reverse(download_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': self.doc.id}) + self.doc_url = reverse( + download_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": self.doc.id, + }, + ) return name def test_views_with_unavailable_id_string(self): path = os.path.join( - self.this_directory, "fixtures", "transportation", - 'transportation.xlsx' + self.this_directory, "fixtures", "transportation", "transportation.xlsx" ) - with open(path, 'rb') as doc_file: + with open(path, "rb") as doc_file: self.post_data = {} - self.post_data['doc'] = doc_file + self.post_data["doc"] = doc_file self.client.post(self.edit_url, self.post_data) self.doc = MetaData.objects.all().reverse()[0] - download_metadata_url = reverse(download_metadata, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string', - 'data_id': self.doc.id}) + download_metadata_url = reverse( + download_metadata, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + "data_id": self.doc.id, + }, + ) response = self.client.get(download_metadata_url) self.assertEqual(response.status_code, 404) - delete_metadata_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string', - 'data_id': self.doc.id}) + delete_metadata_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + "data_id": self.doc.id, + }, + ) - response = self.client.get(delete_metadata_url + '?del=true') + response = self.client.get(delete_metadata_url + "?del=true") self.assertEqual(response.status_code, 404) def test_adds_supporting_doc_on_submit(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='supporting_doc')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="supporting_doc") + ) self._add_metadata() - self.assertEqual(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='supporting_doc'))) + self.assertEqual( + count + 1, + len( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ) + ), + ) def test_delete_cached_xform_metadata_object_on_save(self): count = MetaData.objects.count() - cache.set('{}{}'.format(XFORM_METADATA_CACHE, self.xform.id), True) + cache.set("{}{}".format(XFORM_METADATA_CACHE, self.xform.id), True) self._add_metadata() - self.assertIsNone( - cache.get('{}{}'.format(XFORM_METADATA_CACHE, self.xform.id))) + self.assertIsNone(cache.get("{}{}".format(XFORM_METADATA_CACHE, self.xform.id))) self.assertEqual(count + 1, MetaData.objects.count()) def test_adds_supporting_media_on_submit(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='media')) - self._add_metadata(data_type='media') - self.assertEqual(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='media'))) + count = len(MetaData.objects.filter(object_id=self.xform.id, data_type="media")) + self._add_metadata(data_type="media") + self.assertEqual( + count + 1, + len(MetaData.objects.filter(object_id=self.xform.id, data_type="media")), + ) def test_adds_mapbox_layer_on_submit(self): count = MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count() + object_id=self.xform.id, data_type="mapbox_layer" + ).count() self.post_data = {} - self.post_data['map_name'] = 'test_mapbox_layer' - self.post_data['link'] = 'http://0.0.0.0:8080' + self.post_data["map_name"] = "test_mapbox_layer" + self.post_data["link"] = "http://0.0.0.0:8080" self.client.post(self.edit_url, self.post_data) - self.assertEqual(count + 1, MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count()) + self.assertEqual( + count + 1, + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + ) def test_shows_supporting_doc_after_submit(self): name = self._add_metadata() @@ -128,7 +167,7 @@ def test_shows_supporting_doc_after_submit(self): self.assertContains(response, name) def test_shows_supporting_media_after_submit(self): - name = self._add_metadata(data_type='media') + name = self._add_metadata(data_type="media") response = self.client.get(self.url) self.assertContains(response, name) self.xform.shared = True @@ -139,23 +178,23 @@ def test_shows_supporting_media_after_submit(self): def test_shows_mapbox_layer_after_submit(self): self.post_data = {} - self.post_data['map_name'] = 'test_mapbox_layer' - self.post_data['link'] = 'http://0.0.0.0:8080' + self.post_data["map_name"] = "test_mapbox_layer" + self.post_data["link"] = "http://0.0.0.0:8080" response = self.client.post(self.edit_url, self.post_data) response = self.client.get(self.url) - self.assertContains(response, 'test_mapbox_layer') + self.assertContains(response, "test_mapbox_layer") self.xform.shared = True self.xform.save() response = self.anon.get(self.url) self.assertEqual(response.status_code, 200) - self.assertContains(response, 'test_mapbox_layer') + self.assertContains(response, "test_mapbox_layer") def test_download_supporting_doc(self): self._add_metadata() response = self.client.get(self.doc_url) self.assertEqual(response.status_code, 200) - fileName, ext = os.path.splitext(response['Content-Disposition']) - self.assertEqual(ext, '.xlsx') + fileName, ext = os.path.splitext(response["Content-Disposition"]) + self.assertEqual(ext, ".xlsx") def test_no_download_supporting_doc_for_anon(self): self._add_metadata() @@ -163,11 +202,11 @@ def test_no_download_supporting_doc_for_anon(self): self.assertEqual(response.status_code, 403) def test_download_supporting_media(self): - self._add_metadata(data_type='media') + self._add_metadata(data_type="media") response = self.client.get(self.doc_url) self.assertEqual(response.status_code, 200) - fileName, ext = os.path.splitext(response['Content-Disposition']) - self.assertEqual(ext, '.png') + fileName, ext = os.path.splitext(response["Content-Disposition"]) + self.assertEqual(ext, ".png") def test_shared_download_supporting_doc_for_anon(self): self._add_metadata() @@ -177,151 +216,191 @@ def test_shared_download_supporting_doc_for_anon(self): self.assertEqual(response.status_code, 200) def test_shared_download_supporting_media_for_anon(self): - self._add_metadata(data_type='media') + self._add_metadata(data_type="media") self.xform.shared = True self.xform.save() response = self.anon.get(self.doc_url) self.assertEqual(response.status_code, 200) def test_delete_supporting_doc(self): - count = MetaData.objects.filter(object_id=self.xform.id, - data_type='supporting_doc').count() + count = MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count() self._add_metadata() - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='supporting_doc').count(), count + 1) - doc = MetaData.objects.filter(data_type='supporting_doc').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.client.get(self.delete_doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='supporting_doc').count(), count) + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count(), + count + 1, + ) + doc = MetaData.objects.filter(data_type="supporting_doc").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.client.get(self.delete_doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="supporting_doc" + ).count(), + count, + ) self.assertEqual(response.status_code, 302) def test_delete_supporting_media(self): count = MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count() - self._add_metadata(data_type='media') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count + 1) - response = self.client.get(self.doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count) + object_id=self.xform.id, data_type="media" + ).count() + self._add_metadata(data_type="media") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count + 1, + ) + response = self.client.get(self.doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count, + ) self.assertEqual(response.status_code, 302) - self._add_metadata(data_type='media') - response = self.anon.get(self.doc_url + '?del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, data_type='media').count(), count + 1) + self._add_metadata(data_type="media") + response = self.anon.get(self.doc_url + "?del=true") + self.assertEqual( + MetaData.objects.filter(object_id=self.xform.id, data_type="media").count(), + count + 1, + ) self.assertEqual(response.status_code, 403) def _add_mapbox_layer(self): # check mapbox_layer metadata count self.count = MetaData.objects.filter( - object_id=self.xform.id, data_type='mapbox_layer').count() + object_id=self.xform.id, data_type="mapbox_layer" + ).count() # add mapbox_layer metadata - post_data = {'map_name': 'test_mapbox_layer', - 'link': 'http://0.0.0.0:8080'} + post_data = {"map_name": "test_mapbox_layer", "link": "http://0.0.0.0:8080"} response = self.client.post(self.edit_url, post_data) self.assertEqual(response.status_code, 302) - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count + 1) + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count + 1, + ) def test_delete_mapbox_layer(self): self._add_mapbox_layer() # delete mapbox_layer metadata - doc = MetaData.objects.filter(data_type='mapbox_layer').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.client.get(self.delete_doc_url + '?map_name_del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count) + doc = MetaData.objects.filter(data_type="mapbox_layer").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.client.get(self.delete_doc_url + "?map_name_del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count, + ) self.assertEqual(response.status_code, 302) def test_anon_delete_mapbox_layer(self): self._add_mapbox_layer() - doc = MetaData.objects.filter(data_type='mapbox_layer').reverse()[0] - self.delete_doc_url = reverse(delete_metadata, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': doc.id}) - response = self.anon.get(self.delete_doc_url + '?map_name_del=true') - self.assertEqual(MetaData.objects.filter( - object_id=self.xform.id, - data_type='mapbox_layer').count(), self.count + 1) + doc = MetaData.objects.filter(data_type="mapbox_layer").reverse()[0] + self.delete_doc_url = reverse( + delete_metadata, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": doc.id, + }, + ) + response = self.anon.get(self.delete_doc_url + "?map_name_del=true") + self.assertEqual( + MetaData.objects.filter( + object_id=self.xform.id, data_type="mapbox_layer" + ).count(), + self.count + 1, + ) self.assertEqual(response.status_code, 302) def test_user_source_edit_updates(self): - desc = 'Snooky' - response = self.client.post(self.edit_url, {'source': desc}, - HTTP_X_REQUESTED_WITH='XMLHttpRequest') + desc = "Snooky" + response = self.client.post( + self.edit_url, {"source": desc}, HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) self.assertEqual(response.status_code, 200) self.assertEqual(MetaData.source(self.xform).data_value, desc) def test_upload_source_file(self): - self._add_metadata('source') + self._add_metadata("source") self.assertNotEqual(MetaData.source(self.xform).data_file, None) def test_upload_source_file_set_value_to_name(self): - name = self._add_metadata('source') + name = self._add_metadata("source") self.assertEqual(MetaData.source(self.xform).data_value, name) def test_upload_source_file_keep_name(self): - desc = 'Snooky' - response = self.client.post(self.edit_url, {'source': desc}, - HTTP_X_REQUESTED_WITH='XMLHttpRequest') + desc = "Snooky" + response = self.client.post( + self.edit_url, {"source": desc}, HTTP_X_REQUESTED_WITH="XMLHttpRequest" + ) self.assertEqual(response.status_code, 200) - self._add_metadata('source') + self._add_metadata("source") self.assertNotEqual(MetaData.source(self.xform).data_file, None) self.assertEqual(MetaData.source(self.xform).data_value, desc) def test_media_file_hash(self): name = "screenshot.png" media_file = os.path.join( - self.this_directory, 'fixtures', 'transportation', name) + self.this_directory, "fixtures", "transportation", name + ) content_type = ContentType.objects.get_for_model(self.xform) m = MetaData.objects.create( content_type=content_type, - data_type='media', + data_type="media", object_id=self.xform.id, data_value=name, - data_file=File(open(media_file, 'rb'), name), - data_file_type='image/png') - f = open(media_file, 'rb') - media_hash = 'md5:%s' % hashlib.md5(f.read()).hexdigest() + data_file=File(open(media_file, "rb"), name), + data_file_type="image/png", + ) + f = open(media_file, "rb") + media_hash = "md5:%s" % hashlib.md5(f.read()).hexdigest() f.close() meta_hash = m.hash self.assertEqual(meta_hash, media_hash) self.assertEqual(m.file_hash, media_hash) def test_add_media_url(self): - uri = 'https://devtrac.ona.io/fieldtrips.csv' - count = MetaData.objects.filter(data_type='media').count() - self.client.post(self.edit_url, {'media_url': uri}) - self.assertEqual(count + 1, - len(MetaData.objects.filter(data_type='media'))) + uri = "https://devtrac.ona.io/fieldtrips.csv" + count = MetaData.objects.filter(data_type="media").count() + self.client.post(self.edit_url, {"media_url": uri}) + self.assertEqual(count + 1, len(MetaData.objects.filter(data_type="media"))) def test_windows_csv_file_upload(self): - count = MetaData.objects.filter(data_type='media').count() + count = MetaData.objects.filter(data_type="media").count() media_file = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation.csv') - f = InMemoryUploadedFile(open(media_file, 'rb'), - 'media', - 'transportation.csv', - 'application/octet-stream', - 2625, - None) + self.this_directory, "fixtures", "transportation", "transportation.csv" + ) + f = InMemoryUploadedFile( + open(media_file, "rb"), + "media", + "transportation.csv", + "application/octet-stream", + 2625, + None, + ) MetaData.media_upload(self.xform, f) - media_list = MetaData.objects.filter(data_type='media') + media_list = MetaData.objects.filter(data_type="media") new_count = media_list.count() self.assertEqual(count + 1, new_count) - media = media_list.get(data_value='transportation.csv') - self.assertEqual(media.data_file_type, 'text/csv') + media = media_list.get(data_value="transportation.csv") + self.assertEqual(media.data_file_type, "text/csv") diff --git a/onadata/apps/main/tests/test_metadata.py b/onadata/apps/main/tests/test_metadata.py index 3e37a5ec4f..e038c3db88 100644 --- a/onadata/apps/main/tests/test_metadata.py +++ b/onadata/apps/main/tests/test_metadata.py @@ -1,15 +1,23 @@ - +# -*- coding: utf-8 -*- +""" +Test MetaData model. +""" from onadata.apps.logger.models import Instance, Project, XForm -from onadata.apps.main.models.meta_data import ( - MetaData, - unique_type_for_form, - upload_to) +from onadata.apps.main.models.meta_data import MetaData, unique_type_for_form, upload_to from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.common_tags import GOOGLE_SHEET_DATA_TYPE,\ - GOOGLE_SHEET_ID, USER_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA +from onadata.libs.utils.common_tags import ( + GOOGLE_SHEET_DATA_TYPE, + GOOGLE_SHEET_ID, + UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, + USER_ID, +) +# pylint: disable=too-many-public-methods class TestMetaData(TestBase): + """ + Test MetaData model. + """ def setUp(self): TestBase.setUp(self) @@ -17,55 +25,82 @@ def setUp(self): self._publish_transportation_form_and_submit_instance() def test_create_metadata(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='enketo_url')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ) enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) - self.assertEqual(count + 1, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count + 1, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) def test_create_google_sheet_metadata_object(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type=GOOGLE_SHEET_DATA_TYPE)) - google_sheets_actions = ( - '{} ABC100| ' - '{} True | ' - '{} 123' - ).format(GOOGLE_SHEET_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, USER_ID) + count = len( + MetaData.objects.filter( + object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE + ) + ) + google_sheets_actions = ("{} ABC100| " "{} True | " "{} 123").format( + GOOGLE_SHEET_ID, UPDATE_OR_DELETE_GOOGLE_SHEET_DATA, USER_ID + ) MetaData.set_google_sheet_details(self.xform, google_sheets_actions) # change - self.assertEqual(count + 1, MetaData.objects.filter( - object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE).count()) + self.assertEqual( + count + 1, + MetaData.objects.filter( + object_id=self.xform.id, data_type=GOOGLE_SHEET_DATA_TYPE + ).count(), + ) gsheet_details = MetaData.get_google_sheet_details(self.xform.pk) - self.assertEqual({ - GOOGLE_SHEET_ID: 'ABC100', - UPDATE_OR_DELETE_GOOGLE_SHEET_DATA: 'True', - USER_ID: '123'}, gsheet_details) + self.assertEqual( + { + GOOGLE_SHEET_ID: "ABC100", + UPDATE_OR_DELETE_GOOGLE_SHEET_DATA: "True", + USER_ID: "123", + }, + gsheet_details, + ) def test_saving_same_metadata_object_doesnt_trigger_integrity_error(self): - count = len(MetaData.objects.filter(object_id=self.xform.id, - data_type='enketo_url')) + count = len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ) enketo_url = "https://dmfrm.enketo.org/webform" MetaData.enketo_url(self.xform, enketo_url) count += 1 - self.assertEqual(count, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) MetaData.enketo_url(self.xform, enketo_url) - self.assertEqual(count, len(MetaData.objects.filter( - object_id=self.xform.id, data_type='enketo_url'))) + self.assertEqual( + count, + len( + MetaData.objects.filter(object_id=self.xform.id, data_type="enketo_url") + ), + ) def test_unique_type_for_form(self): metadata = unique_type_for_form( - self.xform, data_type='enketo_url', - data_value="https://dmfrm.enketo.org/webform") + self.xform, + data_type="enketo_url", + data_value="https://dmfrm.enketo.org/webform", + ) self.assertIsInstance(metadata, MetaData) metadata_1 = unique_type_for_form( - self.xform, data_type='enketo_url', - data_value="https://dmerm.enketo.org/webform") + self.xform, + data_type="enketo_url", + data_value="https://dmerm.enketo.org/webform", + ) self.assertIsInstance(metadata_1, MetaData) self.assertNotEqual(metadata.data_value, metadata_1.data_value) @@ -77,18 +112,18 @@ def test_upload_to_with_anonymous_user(self): metadata = MetaData(data_type="media") metadata.content_object = instance filename = "filename" - self.assertEqual(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) # test instance with anonymous user instance_without_user = Instance(xform=self.xform) metadata.content_object = instance_without_user - self.assertEqual(upload_to(metadata, filename), - "{}/{}/{}".format(self.xform.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.xform.user.username, "formid-media", filename), + ) def test_upload_to_with_project_and_xform_instance(self): model_instance = Project(created_by=self.user) @@ -97,17 +132,17 @@ def test_upload_to_with_project_and_xform_instance(self): filename = "filename" - self.assertEqual(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) model_instance = XForm(user=self.user, created_by=self.user) metadata = MetaData(data_type="media") metadata.content_object = model_instance filename = "filename" - self.assertEqual(upload_to(metadata, filename), - "{}/{}/{}".format(self.user.username, - 'formid-media', - filename)) + self.assertEqual( + upload_to(metadata, filename), + "{}/{}/{}".format(self.user.username, "formid-media", filename), + ) diff --git a/onadata/apps/restservice/tests/test_restservice.py b/onadata/apps/restservice/tests/test_restservice.py index 3a4677e26e..9c525714f0 100644 --- a/onadata/apps/restservice/tests/test_restservice.py +++ b/onadata/apps/restservice/tests/test_restservice.py @@ -1,29 +1,37 @@ +# -*- coding: utf-8 -*- +""" +Test RestService model +""" import os import time from django.test.utils import override_settings from django.urls import reverse + from mock import patch from onadata.apps.logger.models.xform import XForm from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.views import show -from onadata.apps.restservice.RestServiceInterface import RestServiceInterface from onadata.apps.restservice.models import RestService +from onadata.apps.restservice.RestServiceInterface import RestServiceInterface from onadata.apps.restservice.services.textit import ServiceDefinition from onadata.apps.restservice.views import add_service, delete_service class RestServiceTest(TestBase): + """ + Test RestService model + """ def setUp(self): - self.service_url = u'http://0.0.0.0:8001/%(id_string)s/post/%(uuid)s' - self.service_name = u'f2dhis2' + self.service_url = "http://0.0.0.0:8001/%(id_string)s/post/%(uuid)s" + self.service_name = "f2dhis2" self._create_user_and_login() - filename = u'dhisform.xlsx' + filename = "dhisform.xlsx" self.this_directory = os.path.dirname(__file__) - path = os.path.join(self.this_directory, u'fixtures', filename) + path = os.path.join(self.this_directory, "fixtures", filename) self._publish_xls_file(path) self.xform = XForm.objects.all().reverse()[0] @@ -31,30 +39,33 @@ def wait(self, t=1): time.sleep(t) def _create_rest_service(self): - rs = RestService(service_url=self.service_url, - xform=self.xform, name=self.service_name) + rs = RestService( + service_url=self.service_url, xform=self.xform, name=self.service_name + ) rs.save() self.restservice = rs def _add_rest_service(self, service_url, service_name): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + add_service_url = reverse( + add_service, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(add_service_url, {}) count = RestService.objects.all().count() self.assertEqual(response.status_code, 200) - post_data = {'service_url': service_url, - 'service_name': service_name} + post_data = {"service_url": service_url, "service_name": service_name} response = self.client.post(add_service_url, post_data) self.assertEqual(response.status_code, 200) self.assertEqual(RestService.objects.all().count(), count + 1) def add_rest_service_with_usename_and_id_string_in_uppercase(self): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper() - }) + add_service_url = reverse( + add_service, + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(add_service_url, {}) self.assertEqual(response.status_code, 200) @@ -75,42 +86,44 @@ def test_anon_service_view(self): self.xform.shared = True self.xform.save() self._logout() - url = reverse(show, kwargs={ - 'username': self.xform.user.username, - 'id_string': self.xform.id_string - }) + url = reverse( + show, + kwargs={ + "username": self.xform.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertNotContains( response, '

    Rest Services

    ') + '"#restservice_tab">Rest Services', + ) def test_delete_service(self): self._add_rest_service(self.service_url, self.service_name) count = RestService.objects.all().count() service = RestService.objects.reverse()[0] - post_data = {'service-id': service.pk} - del_service_url = reverse(delete_service, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + post_data = {"service-id": service.pk} + del_service_url = reverse( + delete_service, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.post(del_service_url, post_data) self.assertEqual(response.status_code, 200) - self.assertEqual( - RestService.objects.all().count(), count - 1 - ) + self.assertEqual(RestService.objects.all().count(), count - 1) def test_add_rest_service_with_wrong_id_string(self): - add_service_url = reverse(add_service, kwargs={ - 'username': self.user.username, - 'id_string': 'wrong_id_string'}) - post_data = {'service_url': self.service_url, - 'service_name': self.service_name} + add_service_url = reverse( + add_service, + kwargs={"username": self.user.username, "id_string": "wrong_id_string"}, + ) + post_data = {"service_url": self.service_url, "service_name": self.service_name} response = self.client.post(add_service_url, post_data) self.assertEqual(response.status_code, 404) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_textit_service(self, mock_http): service_url = "https://textit.io/api/v1/runs.json" service_name = "textit" @@ -123,13 +136,13 @@ def test_textit_service(self, mock_http): default_contact = "sadlsdfskjdfds" MetaData.textit( - self.xform, data_value="{}|{}|{}".format(api_token, - flow_uuid, - default_contact)) + self.xform, + data_value="{}|{}|{}".format(api_token, flow_uuid, default_contact), + ) - xml_submission = os.path.join(self.this_directory, - u'fixtures', - u'dhisform_submission1.xml') + xml_submission = os.path.join( + self.this_directory, "fixtures", "dhisform_submission1.xml" + ) self.assertFalse(mock_http.called) self._make_submission(xml_submission) @@ -137,11 +150,11 @@ def test_textit_service(self, mock_http): self.assertEqual(mock_http.call_count, 1) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_rest_service_not_set(self, mock_http): - xml_submission = os.path.join(self.this_directory, - u'fixtures', - u'dhisform_submission1.xml') + xml_submission = os.path.join( + self.this_directory, "fixtures", "dhisform_submission1.xml" + ) self.assertFalse(mock_http.called) self._make_submission(xml_submission) @@ -155,14 +168,13 @@ def test_clean_keys_of_slashes(self): "hh/group/data_set": "22", "empty_column": "", "false_column": False, - "zero_column": 0 + "zero_column": 0, } expected_data = { "hh_group_data_set": "22", "false_column": "False", - "zero_column": "0" + "zero_column": "0", } - self.assertEqual(expected_data, - service.clean_keys_of_slashes(test_data)) + self.assertEqual(expected_data, service.clean_keys_of_slashes(test_data)) diff --git a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py index 189bcc70a5..1900e12930 100644 --- a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py +++ b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py @@ -1,25 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Test /restservices API endpoint implementation. +""" from django.test.utils import override_settings + from mock import patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.main.models.meta_data import MetaData from onadata.apps.restservice.models import RestService -from onadata.apps.restservice.viewsets.restservices_viewset import \ - RestServicesViewSet +from onadata.apps.restservice.viewsets.restservices_viewset import RestServicesViewSet class TestRestServicesViewSet(TestAbstractViewSet): + """ + Test /restservices API endpoint implementation. + """ def setUp(self): super(TestRestServicesViewSet, self).setUp() - self.view = RestServicesViewSet.as_view({ - 'delete': 'destroy', - 'get': 'retrieve', - 'post': 'create', - 'put': 'update', - 'patch': 'partial_update' - }) + self.view = RestServicesViewSet.as_view( + { + "delete": "destroy", + "get": "retrieve", + "post": "create", + "put": "update", + "patch": "partial_update", + } + ) self._publish_xls_form_to_project() def test_create(self): @@ -28,9 +36,9 @@ def test_create(self): post_data = { "name": "generic_json", "service_url": "https://textit.io", - "xform": self.xform.pk + "xform": self.xform.pk, } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -42,7 +50,7 @@ def test_textit_service_missing_params(self): "service_url": "https://textit.io", "xform": self.xform.pk, } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) @@ -56,33 +64,32 @@ def _create_textit_service(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) self.assertEqual(count + 1, RestService.objects.all().count()) - meta = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit') + meta = MetaData.objects.filter(object_id=self.xform.id, data_type="textit") self.assertEqual(len(meta), 1) rs = RestService.objects.last() expected_dict = { - 'name': u'textit', - 'contacts': u'ksadaskjdajsda', - 'auth_token': u'sadsdfhsdf', - 'flow_uuid': u'sdfskhfskdjhfs', - 'service_url': u'https://textit.io', - 'id': rs.pk, - 'xform': self.xform.pk, - 'active': True, - 'inactive_reason': '', - 'flow_title': '' + "name": "textit", + "contacts": "ksadaskjdajsda", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "service_url": "https://textit.io", + "id": rs.pk, + "xform": self.xform.pk, + "active": True, + "inactive_reason": "", + "flow_title": "", } - response.data.pop('date_modified') - response.data.pop('date_created') + response.data.pop("date_modified") + response.data.pop("date_created") self.assertEqual(response.data, expected_dict) @@ -94,46 +101,48 @@ def test_create_textit_service(self): def test_retrieve_textit_services(self): response_data = self._create_textit_service() - _id = response_data.get('id') + _id = response_data.get("id") - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=_id) expected_dict = { - 'name': u'textit', - 'contacts': u'ksadaskjdajsda', - 'auth_token': u'sadsdfhsdf', - 'flow_uuid': u'sdfskhfskdjhfs', - 'service_url': u'https://textit.io', - 'id': _id, - 'xform': self.xform.pk, - 'active': True, - 'inactive_reason': '', - 'flow_title': '' + "name": "textit", + "contacts": "ksadaskjdajsda", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "service_url": "https://textit.io", + "id": _id, + "xform": self.xform.pk, + "active": True, + "inactive_reason": "", + "flow_title": "", } - response.data.pop('date_modified') - response.data.pop('date_created') + response.data.pop("date_modified") + response.data.pop("date_created") self.assertEqual(response.data, expected_dict) def test_delete_textit_service(self): rest = self._create_textit_service() count = RestService.objects.all().count() - meta_count = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit').count() + meta_count = MetaData.objects.filter( + object_id=self.xform.id, data_type="textit" + ).count() - request = self.factory.delete('/', **self.extra) - response = self.view(request, pk=rest['id']) + request = self.factory.delete("/", **self.extra) + response = self.view(request, pk=rest["id"]) self.assertEqual(response.status_code, 204) self.assertEqual(count - 1, RestService.objects.all().count()) - a_meta_count = MetaData.objects.filter(object_id=self.xform.id, - data_type='textit').count() + a_meta_count = MetaData.objects.filter( + object_id=self.xform.id, data_type="textit" + ).count() self.assertEqual(meta_count - 1, a_meta_count) def test_update(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() post_data = { @@ -143,32 +152,32 @@ def test_update(self): "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", "contacts": "ksadaskjdajsda", - "flow_title": "test-flow" + "flow_title": "test-flow", } - request = self.factory.put('/', data=post_data, **self.extra) + request = self.factory.put("/", data=post_data, **self.extra) response = self.view(request, pk=rest.pk) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['name'], "textit") - self.assertEqual(response.data['flow_title'], 'test-flow') + self.assertEqual(response.data["name"], "textit") + self.assertEqual(response.data["flow_title"], "test-flow") metadata_count = MetaData.objects.count() # Flow title can be updated put_data = { - 'flow_title': 'new-name', - 'xform': self.xform.pk, - 'name': 'textit', - 'service_url': 'https://textit.io', - 'auth_token': 'sadsdfhsdf', - 'flow_uuid': 'sdfskhfskdjhfs', - 'contacts': 'ksadaskjdajsda', + "flow_title": "new-name", + "xform": self.xform.pk, + "name": "textit", + "service_url": "https://textit.io", + "auth_token": "sadsdfhsdf", + "flow_uuid": "sdfskhfskdjhfs", + "contacts": "ksadaskjdajsda", } - request = self.factory.put('/', data=put_data, **self.extra) + request = self.factory.put("/", data=put_data, **self.extra) response = self.view(request, pk=rest.pk) self.assertEqual(response.status_code, 200, response.data) - self.assertEqual(response.data['flow_title'], 'new-name') + self.assertEqual(response.data["flow_title"], "new-name") self.assertEqual(MetaData.objects.count(), metadata_count) def test_update_with_errors(self): @@ -177,14 +186,18 @@ def test_update_with_errors(self): data_value = "{}|{}".format("test", "test2") MetaData.textit(self.xform, data_value) - request = self.factory.get('/', **self.extra) - response = self.view(request, pk=rest.get('id')) + request = self.factory.get("/", **self.extra) + response = self.view(request, pk=rest.get("id")) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - [u"Error occurred when loading textit service." - u"Resolve by updating auth_token, flow_uuid and " - u"contacts fields"]) + self.assertEqual( + response.data, + [ + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and " + "contacts fields" + ], + ) post_data = { "name": "textit", @@ -192,47 +205,47 @@ def test_update_with_errors(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.put('/', data=post_data, **self.extra) - response = self.view(request, pk=rest.get('id')) + request = self.factory.put("/", data=post_data, **self.extra) + response = self.view(request, pk=rest.get("id")) self.assertEqual(response.status_code, 200) def test_delete(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() count = RestService.objects.all().count() - request = self.factory.delete('/', **self.extra) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=rest.pk) self.assertEqual(response.status_code, 204) self.assertEqual(count - 1, RestService.objects.all().count()) def test_retrieve(self): - rest = RestService(name="testservice", - service_url="http://serviec.io", - xform=self.xform) + rest = RestService( + name="testservice", service_url="http://serviec.io", xform=self.xform + ) rest.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=rest.pk) data = { - 'id': rest.pk, - 'xform': self.xform.pk, - 'name': u'testservice', - 'service_url': u'http://serviec.io', - 'active': True, - 'inactive_reason': '' + "id": rest.pk, + "xform": self.xform.pk, + "name": "testservice", + "service_url": "http://serviec.io", + "active": True, + "inactive_reason": "", } - response.data.pop('date_modified') - response.data.pop('date_created') + response.data.pop("date_modified") + response.data.pop("date_created") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) @@ -244,9 +257,9 @@ def test_duplicate_rest_service(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 201) @@ -257,25 +270,27 @@ def test_duplicate_rest_service(self): "xform": self.xform.pk, "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_textit_flow(self, mock_http): - rest = RestService(name="textit", - service_url="https://server.io", - xform=self.xform) + rest = RestService( + name="textit", service_url="https://server.io", xform=self.xform + ) rest.save() - MetaData.textit(self.xform, - data_value='{}|{}|{}'.format("sadsdfhsdf", - "sdfskhfskdjhfs", - "ksadaskjdajsda")) + MetaData.textit( + self.xform, + data_value="{}|{}|{}".format( + "sadsdfhsdf", "sdfskhfskdjhfs", "ksadaskjdajsda" + ), + ) self.assertFalse(mock_http.called) self._make_submissions() @@ -284,17 +299,19 @@ def test_textit_flow(self, mock_http): self.assertEqual(mock_http.call_count, 4) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('requests.post') + @patch("requests.post") def test_textit_flow_without_parsed_instances(self, mock_http): - rest = RestService(name="textit", - service_url="https://server.io", - xform=self.xform) + rest = RestService( + name="textit", service_url="https://server.io", xform=self.xform + ) rest.save() - MetaData.textit(self.xform, - data_value='{}|{}|{}'.format("sadsdfhsdf", - "sdfskhfskdjhfs", - "ksadaskjdajsda")) + MetaData.textit( + self.xform, + data_value="{}|{}|{}".format( + "sadsdfhsdf", "sdfskhfskdjhfs", "ksadaskjdajsda" + ), + ) self.assertFalse(mock_http.called) self._make_submissions() self.assertTrue(mock_http.called) @@ -308,11 +325,11 @@ def test_create_rest_service_invalid_form_id(self): "xform": "invalid", "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", - "contacts": "ksadaskjdajsda" + "contacts": "ksadaskjdajsda", } - request = self.factory.post('/', data=post_data, **self.extra) + request = self.factory.post("/", data=post_data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'xform': [u'Invalid form id']}) + self.assertEqual(response.data, {"xform": ["Invalid form id"]}) self.assertEqual(count, RestService.objects.all().count()) diff --git a/onadata/apps/viewer/tests/test_export_list.py b/onadata/apps/viewer/tests/test_export_list.py index 925c22276c..16cf0a8721 100644 --- a/onadata/apps/viewer/tests/test_export_list.py +++ b/onadata/apps/viewer/tests/test_export_list.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test export list view. +""" import os from django.conf import settings @@ -10,20 +14,31 @@ class TestExportList(TestBase): + """ + Test export list view. + """ def setUp(self): - super(TestExportList, self).setUp() + super().setUp() self._publish_transportation_form() survey = self.surveys[0] self._make_submission( os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', survey, survey + '.xml')) + self.this_directory, + "fixtures", + "transportation", + "instances", + survey, + survey + ".xml", + ) + ) def test_unauthorised_users_cannot_export_form_data(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.CSV_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) @@ -33,43 +48,54 @@ def test_unauthorised_users_cannot_export_form_data(self): '', - response.content.decode('utf-8')) + response.content.decode("utf-8"), + ) self.assertEqual(response.status_code, 200) def test_unsupported_type_export(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - 'export_type': 'gdoc'} + kwargs = { + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + "export_type": "gdoc", + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 400) def test_export_data_with_unavailable_id_string(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.CSV_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.ZIP_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) - kwargs = {'username': self.user.username.upper(), - 'id_string': 'random_id_string', - 'export_type': Export.KML_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": "random_id_string", + "export_type": Export.KML_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 404) def test_csv_export_list(self): - kwargs = {'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - 'export_type': Export.CSV_EXPORT} + kwargs = { + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + "export_type": Export.CSV_EXPORT, + } # test csv url = reverse(export_list, kwargs=kwargs) @@ -77,55 +103,67 @@ def test_csv_export_list(self): self.assertEqual(response.status_code, 200) def test_xls_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.XLS_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.XLS_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_kml_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.KML_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.KML_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_csv_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.CSV_ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.CSV_ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_sav_zip_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.SAV_ZIP_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.SAV_ZIP_EXPORT, + } url = reverse(export_list, kwargs=kwargs) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_external_export_list(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.EXTERNAL_EXPORT} - server = 'http://localhost:8080/xls/23fa4c38c0054748a984ffd89021a295' - data_value = 'template 1 |{0}'.format(server) + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.EXTERNAL_EXPORT, + } + server = "http://localhost:8080/xls/23fa4c38c0054748a984ffd89021a295" + data_value = "template 1 |{0}".format(server) meta = MetaData.external_export(self.xform, data_value) custom_params = { - 'meta': meta.id, + "meta": meta.id, } url = reverse(export_list, kwargs=kwargs) count = len(Export.objects.all()) @@ -135,115 +173,139 @@ def test_external_export_list(self): self.assertEqual(count + 1, count1) def test_external_export_list_no_template(self): - kwargs = {'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': Export.EXTERNAL_EXPORT} + kwargs = { + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": Export.EXTERNAL_EXPORT, + } url = reverse(export_list, kwargs=kwargs) count = len(Export.objects.all()) response = self.client.get(url) self.assertEqual(response.status_code, 403) - self.assertEqual(response.content.decode('utf-8'), - u'No XLS Template set.') + self.assertEqual(response.content.decode("utf-8"), "No XLS Template set.") count1 = len(Export.objects.all()) self.assertEqual(count, count1) class TestDataExportURL(TestBase): - def setUp(self): super(TestDataExportURL, self).setUp() self._publish_transportation_form() def _filename_from_disposition(self, content_disposition): - filename_pos = content_disposition.index('filename=') + filename_pos = content_disposition.index("filename=") self.assertTrue(filename_pos != -1) - return content_disposition[filename_pos + len('filename='):] + return content_disposition[filename_pos + len("filename=") :] def test_csv_export_url(self): self._submit_transport_instance() - url = reverse('csv_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "csv_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") def test_csv_export_url_without_records(self): # this has been refactored so that if NoRecordsFound Exception is # thrown, it will return an empty csv containing only the xform schema - url = reverse('csv_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "csv_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) # Unpack response streaming data - export_data = [i.decode( - 'utf-8').replace('\n', '').split( - ',') for i in response.streaming_content] + export_data = [ + i.decode("utf-8").replace("\n", "").split(",") + for i in response.streaming_content + ] xform_headers = self.xform.get_headers() # Remove review headers from xform headers - for x in ['_review_status', '_review_comment']: + for x in ["_review_status", "_review_comment"]: xform_headers.remove(x) # Test export data returned is xform headers list self.assertEqual(xform_headers, export_data[0]) def test_xls_export_url(self): self._submit_transport_instance() - url = reverse('xls_export', kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - }) + url = reverse( + "xls_export", + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], - 'application/vnd.openxmlformats') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/vnd.openxmlformats") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.xlsx') + self.assertEqual(ext, ".xlsx") def test_csv_zip_export_url(self): self._submit_transport_instance() - url = reverse('csv_zip_export', kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper(), - }) + url = reverse( + "csv_zip_export", + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + self.assertEqual(ext, ".zip") def test_sav_zip_export_url(self): - filename = os.path.join(settings.PROJECT_ROOT, 'apps', 'logger', - 'tests', 'fixtures', 'childrens_survey.xlsx') + filename = os.path.join( + settings.PROJECT_ROOT, + "apps", + "logger", + "tests", + "fixtures", + "childrens_survey.xlsx", + ) self._publish_xls_file_and_set_xform(filename) - url = reverse('sav_zip_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "sav_zip_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/zip') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/zip") + content_disposition = headers["Content-Disposition"] filename = self._filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.zip') + self.assertEqual(ext, ".zip") def test_sav_zip_export_long_variable_length(self): self._submit_transport_instance() - url = reverse('sav_zip_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - }) + url = reverse( + "sav_zip_export", + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) diff --git a/onadata/libs/tests/serializers/test_metadata_serializer.py b/onadata/libs/tests/serializers/test_metadata_serializer.py index 54f0d7e3fb..79da930241 100644 --- a/onadata/libs/tests/serializers/test_metadata_serializer.py +++ b/onadata/libs/tests/serializers/test_metadata_serializer.py @@ -3,11 +3,11 @@ Test onadata.libs.serializers.metadata_serializer """ import os + from django.core.files.uploadedfile import InMemoryUploadedFile from django.test.utils import override_settings -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.libs.serializers.metadata_serializer import MetaDataSerializer @@ -23,8 +23,7 @@ def test_data_value_is_required(self): data = {} serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) - self.assertEqual(serializer.errors['data_value'], - [u'This field is required.']) + self.assertEqual(serializer.errors["data_value"], ["This field is required."]) def test_media_url_validation(self): """ @@ -33,16 +32,21 @@ def test_media_url_validation(self): self._login_user_and_profile() self._publish_form_with_hxl_support() data = { - 'data_value': 'http://example.com', - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": "http://example.com", + "data_type": "media", + "xform": self.xform.pk, } serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) self.assertEqual( - serializer.errors['data_value'], - [(u"Cannot get filename from URL %(data_value)s. URL should " - u"include the filename e.g %(data_value)s/data.csv" % data)]) + serializer.errors["data_value"], + [ + ( + "Cannot get filename from URL %(data_value)s. URL should " + "include the filename e.g %(data_value)s/data.csv" % data + ) + ], + ) @override_settings(SUPPORTED_MEDIA_UPLOAD_TYPES=[]) def test_unsupported_media_files(self): @@ -51,22 +55,24 @@ def test_unsupported_media_files(self): """ self._login_user_and_profile() self._publish_form_with_hxl_support() - data_value = 'sample.svg' - path = os.path.join(os.path.dirname(__file__), 'fixtures', - 'sample.svg') + data_value = "sample.svg" + path = os.path.join(os.path.dirname(__file__), "fixtures", "sample.svg") with open(path) as f: f = InMemoryUploadedFile( - f, 'media', data_value, 'application/octet-stream', 2324, None) + f, "media", data_value, "application/octet-stream", 2324, None + ) data = { - 'data_value': data_value, - 'data_file': f, - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": data_value, + "data_file": f, + "data_type": "media", + "xform": self.xform.pk, } serializer = MetaDataSerializer(data=data) self.assertFalse(serializer.is_valid()) - self.assertEqual(serializer.errors['data_file'], - [("Unsupported media file type image/svg+xml")]) + self.assertEqual( + serializer.errors["data_file"], + [("Unsupported media file type image/svg+xml")], + ) def test_svg_media_files(self): """ @@ -74,19 +80,20 @@ def test_svg_media_files(self): """ self._login_user_and_profile() self._publish_form_with_hxl_support() - data_value = 'sample.svg' - path = os.path.join(os.path.dirname(__file__), 'fixtures', - 'sample.svg') + data_value = "sample.svg" + path = os.path.join(os.path.dirname(__file__), "fixtures", "sample.svg") with open(path) as f: f = InMemoryUploadedFile( - f, 'media', data_value, 'application/octet-stream', 2324, None) + f, "media", data_value, "application/octet-stream", 2324, None + ) data = { - 'data_value': data_value, - 'data_file': f, - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": data_value, + "data_file": f, + "data_type": "media", + "xform": self.xform.pk, } serializer = MetaDataSerializer(data=data) self.assertTrue(serializer.is_valid()) - self.assertEqual(serializer.validated_data['data_file_type'], - 'image/svg+xml') + self.assertEqual( + serializer.validated_data["data_file_type"], "image/svg+xml" + ) diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index 90b90d4a03..7a8a5710b7 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Test logger_tools utility functions. +""" import os import re from io import BytesIO @@ -11,39 +15,52 @@ from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models import Instance from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.common_tags import (MEDIA_ALL_RECEIVED, MEDIA_COUNT, - TOTAL_MEDIA) +from onadata.libs.utils.common_tags import MEDIA_ALL_RECEIVED, MEDIA_COUNT, TOTAL_MEDIA from onadata.libs.utils.logger_tools import ( - create_instance, generate_content_disposition_header, get_first_record, - safe_create_instance) + create_instance, + generate_content_disposition_header, + get_first_record, + safe_create_instance, +) from onadata.apps.logger.xform_instance_parser import AttachmentNameError from django.core.files.uploadedfile import InMemoryUploadedFile class TestLoggerTools(PyxformTestCase, TestBase): + """ + Test logger_tools utility functions. + """ + def test_generate_content_disposition_header(self): file_name = "export" extension = "ext" - date_pattern = "\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}" # noqa + date_pattern = "\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}" # noqa file_name_pattern = "%s.%s" % (file_name, extension) - file_name_with_timestamp_pattern = \ - "%s-%s.%s" % (file_name, date_pattern, extension) - return_value_with_no_name = \ - generate_content_disposition_header(None, extension) + file_name_with_timestamp_pattern = "%s-%s.%s" % ( + file_name, + date_pattern, + extension, + ) + return_value_with_no_name = generate_content_disposition_header(None, extension) self.assertEqual(return_value_with_no_name, "attachment;") - return_value_with_name_and_no_show_date = \ - generate_content_disposition_header(file_name, extension) + return_value_with_name_and_no_show_date = generate_content_disposition_header( + file_name, extension + ) self.assertTrue( - re.search(file_name_with_timestamp_pattern, - return_value_with_name_and_no_show_date)) + re.search( + file_name_with_timestamp_pattern, + return_value_with_name_and_no_show_date, + ) + ) - return_value_with_name_and_false_show_date = \ + return_value_with_name_and_false_show_date = ( generate_content_disposition_header(file_name, extension, False) + ) self.assertTrue( - re.search(file_name_pattern, - return_value_with_name_and_false_show_date)) + re.search(file_name_pattern, return_value_with_name_and_false_show_date) + ) def test_attachment_tracking(self): """ @@ -68,53 +85,66 @@ def test_attachment_tracking(self): 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 1) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image2", content_type="image/jpeg") + path=file2_path, field_name="image2", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance2.json[TOTAL_MEDIA], 2) self.assertEqual(instance2.json[MEDIA_COUNT], 2) self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) media2_file = django_file( - path=file2_path, field_name="image2", content_type="image/jpeg") + path=file2_path, field_name="image2", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance3 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance3.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance3.json[TOTAL_MEDIA], 2) self.assertEqual(instance3.json[MEDIA_COUNT], 2) self.assertEqual(instance3.json[TOTAL_MEDIA], instance2.total_media) self.assertEqual(instance3.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual(instance3.json[MEDIA_ALL_RECEIVED], - instance3.media_all_received) + self.assertEqual( + instance3.json[MEDIA_ALL_RECEIVED], instance3.media_all_received + ) def test_attachment_tracking_for_repeats(self): """ @@ -144,39 +174,49 @@ def test_attachment_tracking_for_repeats(self): 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 1) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance2.json[TOTAL_MEDIA], 2) self.assertEqual(instance2.json[MEDIA_COUNT], 2) self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_for_nested_repeats(self): """ @@ -208,39 +248,49 @@ def test_attachment_tracking_for_nested_repeats(self): 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 1) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ - "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29" + "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance2.json[TOTAL_MEDIA], 2) self.assertEqual(instance2.json[MEDIA_COUNT], 2) self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_replaced_attachments_not_tracked(self): """ @@ -266,32 +316,40 @@ def test_replaced_attachments_not_tracked(self): Health_2011_03_13.xml_2011-03-15_20-30-28.xml 1300221157303.jpg - """.format(self.xform.id_string) - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + """.format( + self.xform.id_string + ) + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) self.assertTrue(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), - 2) + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 2) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) - patch_value = 'onadata.apps.logger.models.Instance.get_expected_media' + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) + patch_value = "onadata.apps.logger.models.Instance.get_expected_media" with patch(patch_value) as get_expected_media: - get_expected_media.return_value = ['1300375832136.jpg'] + get_expected_media.return_value = ["1300375832136.jpg"] updated_xml_string = """ @@ -301,18 +359,21 @@ def test_replaced_attachments_not_tracked(self): 1300375832136.jpg - """.format(self.xform.id_string) - file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011"\ - "-03-17_16-29-59/1300375832136.jpg".format( - settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file2_path = ( + "{}/apps/logger/tests/Water_2011_03_17_2011" + "-03-17_16-29-59/1300375832136.jpg".format(settings.PROJECT_ROOT) + ) media2_file = django_file( - path=file2_path, - field_name="image1", - content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(updated_xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(updated_xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) @@ -320,13 +381,11 @@ def test_replaced_attachments_not_tracked(self): # Since the file is no longer present in the submission self.assertEqual(instance2.json[TOTAL_MEDIA], 1) self.assertEqual(instance2.json[MEDIA_COUNT], 1) + self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) + self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) self.assertEqual( - instance2.json[TOTAL_MEDIA], instance2.total_media) - self.assertEqual( - instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual( - instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_duplicate(self): """ @@ -350,37 +409,45 @@ def test_attachment_tracking_duplicate(self): 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 1) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) media2_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media2_file], + ) instance2 = Instance.objects.get(pk=instance.pk) self.assertFalse(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance2.json[TOTAL_MEDIA], 2) self.assertEqual(instance2.json[MEDIA_COUNT], 1) self.assertEqual(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEqual(instance2.json[MEDIA_COUNT], instance2.media_count) - self.assertEqual(instance2.json[MEDIA_ALL_RECEIVED], - instance2.media_all_received) + self.assertEqual( + instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received + ) def test_attachment_tracking_not_in_submission(self): """ @@ -403,27 +470,35 @@ def test_attachment_tracking_not_in_submission(self): 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - file2_path = "{}/libs/tests/utils/fixtures/tutorial/instances/uuid1/"\ - "1442323232322.jpg".format(settings.PROJECT_ROOT) + """.format( + self.xform.id_string + ) + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + file2_path = ( + "{}/libs/tests/utils/fixtures/tutorial/instances/uuid1/" + "1442323232322.jpg".format(settings.PROJECT_ROOT) + ) media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + path=file_path, field_name="image1", content_type="image/jpeg" + ) media2_file = django_file( - path=file2_path, field_name="image1", content_type="image/jpeg") + path=file2_path, field_name="image1", content_type="image/jpeg" + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file, media2_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file, media2_file], + ) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual(instance.json[TOTAL_MEDIA], 2) self.assertEqual(instance.json[MEDIA_COUNT], 1) self.assertEqual(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEqual(instance.json[MEDIA_COUNT], instance.media_count) - self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], - instance.media_all_received) + self.assertEqual(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) def test_get_first_record(self): """ @@ -437,18 +512,21 @@ def test_get_first_record(self): self._create_user_and_login() xform = self._publish_markdown(xform_md, self.user) - self.assertIsNone(get_first_record(Instance.objects.all().only('id'))) + self.assertIsNone(get_first_record(Instance.objects.all().only("id"))) xml_string = """ Alice - """.format(xform.id_string) + """.format( + xform.id_string + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[]) - record = get_first_record(Instance.objects.all().only('id')) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[], + ) + record = get_first_record(Instance.objects.all().only("id")) self.assertIsNotNone(record) self.assertEqual(record.id, instance.id) @@ -457,8 +535,10 @@ def test_check_encryption_status(self): Test that the encryption status of a submission is checked and unencrypted submissions are rejected when made to encrypted forms. """ - form_path = (f"{settings.PROJECT_ROOT}/libs/tests/" - "fixtures/tutorial/tutorial_encrypted.xlsx") + form_path = ( + f"{settings.PROJECT_ROOT}/libs/tests/" + "fixtures/tutorial/tutorial_encrypted.xlsx" + ) self._publish_xls_file_and_set_xform(form_path) instance_xml = f""" 1300221157303.jpg 1300375832136.jpg - """.format(self.xform.id_string) + """.format( + self.xform.id_string + ) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - f = open(file_path, 'rb') + file_path = ( + "{}/apps/logger/tests/Health_2011_03_13." + "xml_2011-03-15_20-30-28/1300221157303" + ".jpg".format(settings.PROJECT_ROOT) + ) + f = open(file_path, "rb") media_file = InMemoryUploadedFile( file=f, field_name="image1", - name=f'{f.name} +\ - test_file_name_test_file_name_test_file_name_test_file_name_test_file_name_test_file_name', # noqa + name=f"{f.name} +\ + test_file_name_test_file_name_test_file_name_test_file_name_test_file_name_test_file_name", # noqa content_type="image/jpeg", size=os.path.getsize(file_path), - charset=None + charset=None, ) with self.assertRaises(AttachmentNameError): create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[media_file], + ) diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index deaec9556f..28434b9aa3 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -1,23 +1,22 @@ # -*- coding: utf-8 -*- -"""Util functions for data views.""" +"""Utility functions for data views.""" import json import os -import requests import sys import zipfile -from builtins import open from json.decoder import JSONDecodeError from tempfile import NamedTemporaryFile from typing import Dict from xml.dom import minidom -from six.moves.urllib.parse import urljoin - from django.conf import settings from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile from django.utils.translation import ugettext as _ +import requests +from six.moves.urllib.parse import urljoin + from onadata.libs.exceptions import EnketoError from onadata.libs.utils import common_tags from onadata.libs.utils.common_tags import EXPORT_MIMES @@ -127,16 +126,16 @@ def django_file(path, field_name, content_type): """Return an InMemoryUploadedFile object for file uploads.""" # adapted from here: http://groups.google.com/group/django-users/browse_th\ # read/thread/834f988876ff3c45/ - file_object = open(path, "rb") - - return InMemoryUploadedFile( - file=file_object, - field_name=field_name, - name=file_object.name, - content_type=content_type, - size=os.path.getsize(path), - charset=None, - ) + + with open(path, "rb") as file_object: + return InMemoryUploadedFile( + file=file_object, + field_name=field_name, + name=file_object.name, + content_type=content_type, + size=os.path.getsize(path), + charset=None, + ) def export_def_from_filename(filename): @@ -183,7 +182,7 @@ def get_enketo_urls( "instance": instance_xml, "instance_id": instance_id, # convert to unicode string in python3 compatible way - "return_url": "%s" % return_url, + "return_url": f"{return_url}", } ) @@ -215,20 +214,22 @@ def get_enketo_urls( handle_enketo_error(response) + return None + def handle_enketo_error(response): """Handle enketo error response.""" try: data = json.loads(response.content) - except (ValueError, JSONDecodeError): + except (ValueError, JSONDecodeError) as e: report_exception( - "HTTP Error {}".format(response.status_code), response.text, sys.exc_info() + f"HTTP Error {response.status_code}", response.text, sys.exc_info() ) if response.status_code == 502: raise EnketoError( - "Sorry, we cannot load your form right now. Please try " "again later." - ) - raise EnketoError() + "Sorry, we cannot load your form right now. Please try again later." + ) from e + raise EnketoError() from e else: if "message" in data: raise EnketoError(data["message"]) @@ -243,7 +244,7 @@ def generate_enketo_form_defaults(xform, **kwargs): for (name, value) in kwargs.items(): field = xform.get_survey_element(name) if field: - defaults["defaults[{}]".format(field.get_xpath())] = value + defaults[f"defaults[{field.get_xpath()}]"] = value return defaults @@ -251,30 +252,30 @@ def generate_enketo_form_defaults(xform, **kwargs): def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file - tmp = NamedTemporaryFile() - with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: - for attachment in attachments: - default_storage = get_storage_class()() - filename = attachment.media_file.name - - if default_storage.exists(filename): - try: - with default_storage.open(filename) as f: - if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: - report_exception( - "Create attachment zip exception", - "File is greater than {} bytes".format( - settings.ZIP_REPORT_ATTACHMENT_LIMIT - ), - ) - break - else: + with NamedTemporaryFile() as tmp: + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: + for attachment in attachments: + default_storage = get_storage_class()() + filename = attachment.media_file.name + + if default_storage.exists(filename): + try: + with default_storage.open(filename) as f: + if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: + report_exception( + "Create attachment zip exception", + ( + "File is greater than " + f"{settings.ZIP_REPORT_ATTACHMENT_LIMIT} bytes" + ), + ) + break z.writestr(attachment.media_file.name, f.read()) - except IOError as e: - report_exception("Create attachment zip exception", e) - break + except IOError as e: + report_exception("Create attachment zip exception", e) + break - return tmp + return tmp def get_form(kwargs): @@ -282,9 +283,11 @@ def get_form(kwargs): # adding inline imports here because adding them at the top of the file # triggers the following error: # django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet. - from onadata.apps.logger.models import XForm + # pylint: disable=import-outside-toplevel from django.http import Http404 + from onadata.apps.logger.models import XForm + queryset = kwargs.pop("queryset", XForm.objects.all()) kwargs["deleted_at__isnull"] = True xform = queryset.filter(**kwargs).first() @@ -294,6 +297,7 @@ def get_form(kwargs): raise Http404("XForm does not exist.") +# pylint: disable=too-many-arguments def get_form_url( request, username=None, @@ -317,16 +321,14 @@ def get_form_url( else: http_host = request.META.get("HTTP_HOST", "ona.io") - url = "%s://%s" % (protocol, http_host) + url = f"{protocol}://{http_host}" if preview: url += "/preview" if xform_pk and generate_consistent_urls: - url += "/enketo/{}".format(xform_pk) + url += f"/enketo/{xform_pk}" elif username: - url += ( - "/{}/{}".format(username, xform_pk) if xform_pk else "/{}".format(username) - ) + url += f"/{username}/{xform_pk}" if xform_pk else f"/{username}" return url From 011bd696b6d6caea911630bd0f20f7d90a7de86c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 13:40:34 +0300 Subject: [PATCH 093/234] batch: cleanup --- onadata/apps/api/viewsets/osm_viewset.py | 5 + onadata/apps/logger/models/instance.py | 85 +++++++-- onadata/apps/main/models/user_profile.py | 17 +- onadata/apps/main/tests/test_process.py | 217 +++++++++++++---------- onadata/libs/utils/common_tools.py | 23 ++- onadata/libs/utils/viewer_tools.py | 49 ++--- 6 files changed, 243 insertions(+), 153 deletions(-) diff --git a/onadata/apps/api/viewsets/osm_viewset.py b/onadata/apps/api/viewsets/osm_viewset.py index 085d636fba..28a85b084c 100644 --- a/onadata/apps/api/viewsets/osm_viewset.py +++ b/onadata/apps/api/viewsets/osm_viewset.py @@ -104,6 +104,7 @@ class OsmViewSet( queryset = XForm.objects.filter().select_related() def get_serializer_class(self): + """Returns the OSMSiteMapSerializer class when list API is invoked.""" form_pk = self.kwargs.get("pk") if self.action == "list" and form_pk is None: return OSMSiteMapSerializer @@ -111,6 +112,7 @@ def get_serializer_class(self): return super().get_serializer_class() def filter_queryset(self, queryset): + """Filters the queryset using the ``pk`` when used.""" form_pk = self.kwargs.get("pk") if form_pk: queryset = queryset.filter(pk=form_pk) @@ -118,6 +120,7 @@ def filter_queryset(self, queryset): return super().filter_queryset(queryset) def get_object(self): + """Returns the Instance object using the ``pk`` and ``dataid`` lookup values.""" obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields form_pk = self.kwargs.get(pk_lookup) @@ -134,6 +137,7 @@ def get_object(self): return obj def retrieve(self, request, *args, **kwargs): + """Returns a single Instance JSON object API response""" fmt = kwargs.get("format", request.accepted_renderer.format) if fmt != "osm": pk_lookup, dataid_lookup = self.lookup_fields @@ -159,6 +163,7 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) def list(self, request, *args, **kwargs): + """Returns a list of URLs to the individual XForm OSM data.""" fmt = kwargs.get("format", request.accepted_renderer.format) form_pk = kwargs.get("pk") if form_pk: diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index ce42961708..5f3dc6fac6 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -3,9 +3,7 @@ Instance model class """ import math -import pytz from datetime import datetime -from deprecated import deprecated from django.conf import settings from django.contrib.auth import get_user_model @@ -13,11 +11,14 @@ from django.contrib.gis.geos import GeometryCollection, Point from django.core.cache import cache from django.db import connection, transaction -from django.db.models import Q, JSONField +from django.db.models import Q from django.db.models.signals import post_delete, post_save from django.urls import reverse from django.utils import timezone from django.utils.translation import ugettext as _ + +import pytz +from deprecated import deprecated from taggit.managers import TaggableManager from onadata.apps.logger.models.submission_review import SubmissionReview @@ -56,6 +57,8 @@ MEDIA_COUNT, MONGO_STRFTIME, NOTES, + REVIEW_COMMENT, + REVIEW_DATE, REVIEW_STATUS, START, STATUS, @@ -67,8 +70,6 @@ VERSION, XFORM_ID, XFORM_ID_STRING, - REVIEW_COMMENT, - REVIEW_DATE, ) from onadata.libs.utils.dict_tools import get_values_matching_key from onadata.libs.utils.model_tools import set_uuid @@ -77,10 +78,14 @@ ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False ) +# pylint: disable=invalid-name User = get_user_model() def get_attachment_url(attachment, suffix=None): + """ + Returns the attachment URL for a given suffix + """ kwargs = {"pk": attachment.pk} url = ( f"{reverse('files-detail', kwargs=kwargs)}" @@ -131,6 +136,9 @@ def __str__(self): def numeric_checker(string_value): + """ + Checks if a ``string_value`` is a numeric value. + """ try: return int(string_value) except ValueError: @@ -149,24 +157,28 @@ def numeric_checker(string_value): def get_id_string_from_xml_str(xml_str): + """ + Parses an XML ``xml_str`` and returns the top level id string. + """ xml_obj = clean_and_parse_xml(xml_str) root_node = xml_obj.documentElement id_string = root_node.getAttribute("id") - if len(id_string) == 0: + if id_string: # may be hidden in submission/data/id_string elems = root_node.getElementsByTagName("data") for data in elems: id_string = data.childNodes[0].getAttribute("id") - if len(id_string) > 0: + if id_string: break return id_string def submission_time(): + """Returns current timestamp via timezone.now().""" return timezone.now() @@ -203,6 +215,7 @@ def _update_submission_count_for_today( @app.task @transaction.atomic() def update_xform_submission_count(instance_id, created): + """Updates the XForm submissions count on a new submission being created.""" if created: # pylint: disable=import-outside-toplevel @@ -251,6 +264,7 @@ def update_xform_submission_count(instance_id, created): # pylint: disable=unused-argument,invalid-name def update_xform_submission_count_delete(sender, instance, **kwargs): + """Updates the XForm submissions count on deletion of a submission.""" try: xform = XForm.objects.select_for_update().get(pk=instance.xform.pk) except XForm.DoesNotExist: @@ -304,6 +318,10 @@ def save_full_json(instance_id, created): # pylint: disable=unused-argument @app.task def update_project_date_modified(instance_id, created): + """Update the project's date_modified + + Changes the etag value of the projects endpoint. + """ # update the date modified field of the project which will change # the etag value of the projects endpoint try: @@ -319,6 +337,7 @@ def update_project_date_modified(instance_id, created): def convert_to_serializable_date(date): + """Returns the ISO format of a date object if it has the attribute 'isoformat'.""" if hasattr(date, "isoformat"): return date.isoformat() @@ -330,13 +349,15 @@ class InstanceBaseClass: @property def point(self): + """Returns the Point of the first geom if it is a collection.""" geom_collection = self.geom - if geom_collection and len(geom_collection): + if geom_collection and isinstance(geom_collection, list): return geom_collection[0] - return None + return self.geom def numeric_converter(self, json_dict, numeric_fields=None): + """Converts strings in a python object ``json_dict`` to their numeric value.""" if numeric_fields is None: # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) @@ -378,17 +399,21 @@ def _set_geom(self): xform.instances_with_geopoints = True xform.save() + # pylint: disable=attribute-defined-outside-init self.geom = GeometryCollection(points) def _set_json(self): + # pylint: disable=attribute-defined-outside-init self.json = self.get_full_dict() def get_full_dict(self, load_existing=True): + """Returns the submission XML as a python dictionary object.""" doc = self.json or {} if load_existing else {} # Get latest dict doc = self.get_dict() # pylint: disable=no-member if self.id: + geopoint = ([self.point.y, self.point.x] if self.point else [None, None],) doc.update( { UUID: self.uuid, @@ -402,9 +427,7 @@ def get_full_dict(self, load_existing=True): DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), XFORM_ID: self.xform.pk, - GEOLOCATION: [self.point.y, self.point.x] - if self.point - else [None, None], + GEOLOCATION: geopoint, SUBMITTED_BY: self.user.username if self.user else None, } ) @@ -454,9 +477,11 @@ def get_full_dict(self, load_existing=True): def _set_parser(self): if not hasattr(self, "_parser"): # pylint: disable=no-member + # pylint: disable=attribute-defined-outside-init self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): + # pylint: disable=attribute-defined-outside-init self.survey_type, _created = SurveyType.objects.get_or_create( slug=self.get_root_node_name() ) @@ -471,6 +496,7 @@ def _set_uuid(self): set_uuid(self) def get(self, abbreviated_xpath): + """Returns the XML element at the ``abbreviated_xpath``.""" self._set_parser() return self._parser.get(abbreviated_xpath) @@ -487,6 +513,7 @@ def get_dict(self, force_new=False, flat=True): return self.numeric_converter(instance_dict) def get_notes(self): + """Returns a list of notes.""" # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] @@ -504,14 +531,17 @@ def get_review_status_and_comment(self): return None def get_root_node(self): + """Returns the XML submission's root node.""" self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): + """Returns the XML submission's root node name.""" self._set_parser() return self._parser.get_root_node_name() def get_duration(self): + """Returns the duration between the `start` and `end` questions of a form.""" data = self.get_dict() # pylint: disable=no-member start_name = _get_tag_or_element_type_xpath(self.xform, START) @@ -526,6 +556,7 @@ def get_latest_review(self): Used in favour of `get_review_status_and_comment`. """ try: + # pylint: disable=no-member return self.reviews.latest("date_modified") except SubmissionReview.DoesNotExist: return None @@ -536,7 +567,7 @@ class Instance(models.Model, InstanceBaseClass): Model representing a single submission to an XForm """ - json = JSONField(default=dict, null=False) + json = models.JSONField(default=dict, null=False) xml = models.TextField() user = models.ForeignKey( User, related_name="instances", null=True, on_delete=models.SET_NULL @@ -595,6 +626,7 @@ class Meta: @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now(), user=None): + """Set's the timestamp when a submission was deleted.""" try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: @@ -658,6 +690,7 @@ def num_of_media(self): @property def attachments_count(self): + """Returns the number of attachments a submission has.""" return ( self.attachments.filter(name__in=self.get_expected_media()) .distinct("name") @@ -665,6 +698,7 @@ def attachments_count(self): .count() ) + # pylint: disable=arguments-differ def save(self, *args, **kwargs): force = kwargs.get("force") @@ -684,6 +718,7 @@ def save(self, *args, **kwargs): # pylint: disable=no-member def set_deleted(self, deleted_at=timezone.now(), user=None): + """Set the timestamp and user when a submission is deleted.""" if user: self.deleted_by = user self.deleted_at = deleted_at @@ -705,6 +740,12 @@ def soft_delete_attachments(self, user=None): # pylint: disable=unused-argument def post_save_submission(sender, instance=None, created=False, **kwargs): + """Update XForm, Project, JSON field + + - XForm submission coun + - Project date modified + - Update the submission JSON field data + """ if instance.deleted_at is not None: _update_submission_count_for_today( instance.xform_id, incr=False, date_created=instance.date_created @@ -733,6 +774,8 @@ def post_save_submission(sender, instance=None, created=False, **kwargs): class InstanceHistory(models.Model, InstanceBaseClass): + """Stores deleted submission XML to maintain a history of edits.""" + class Meta: app_label = "logger" @@ -753,65 +796,81 @@ class Meta: @property def xform(self): + """Returns the XForm object linked to this submission.""" return self.xform_instance.xform @property def attachments(self): + """Returns the attachments linked to this submission.""" return self.xform_instance.attachments.all() @property def json(self): + """Returns the XML submission as a python dictionary object.""" return self.get_full_dict(load_existing=False) @property def status(self): + """Returns the submission's status""" return self.xform_instance.status @property def tags(self): + """Returns the tags linked to the submission.""" return self.xform_instance.tags @property def notes(self): + """Returns the notes attached to the submission.""" return self.xform_instance.notes.all() @property def reviews(self): + """Returns the submission reviews.""" return self.xform_instance.reviews.all() @property def version(self): + """Returns the XForm verison for the submission.""" return self.xform_instance.version @property def osm_data(self): + """Returns the OSM data for the submission.""" return self.xform_instance.osm_data @property def deleted_at(self): + """Mutes the deleted_at method for the history record.""" return None @property def total_media(self): + """Returns the number of attachments linked to submission.""" return self.xform_instance.total_media @property def has_a_review(self): + """Returns the value of a submission.has_a_review.""" return self.xform_instance.has_a_review @property def media_count(self): + """Returns the number of media attached to the submission.""" return self.xform_instance.media_count @property def media_all_received(self): + """Returns the value of the submission.media_all_received.""" return self.xform_instance.media_all_received def _set_parser(self): if not hasattr(self, "_parser"): + # pylint: disable=attribute-defined-outside-init self._parser = XFormInstanceParser(self.xml, self.xform_instance.xform) # pylint: disable=unused-argument @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): + """Mutes the set_deleted_at method for the history record.""" return None diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 9a9a549716..755390b8a8 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -2,23 +2,24 @@ """ UserProfile model class """ -import requests from django.conf import settings from django.contrib.auth import get_user_model from django.db import models from django.db.models.signals import post_save, pre_save from django.utils.translation import ugettext_lazy -from guardian.shortcuts import get_perms_for_model, assign_perm -from guardian.models import UserObjectPermissionBase -from guardian.models import GroupObjectPermissionBase + +import requests +from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase +from guardian.shortcuts import assign_perm, get_perms_for_model from rest_framework.authtoken.models import Token -from onadata.libs.utils.country_field import COUNTRIES -from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists + from onadata.apps.main.signals import ( - set_api_permissions, - send_inactive_user_email, send_activation_email, + send_inactive_user_email, + set_api_permissions, ) +from onadata.libs.utils.country_field import COUNTRIES +from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists REQUIRE_AUTHENTICATION = "REQUIRE_ODK_AUTHENTICATION" diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 0ac3388ffe..fb1e772b94 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -1,23 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Test usage process - form publishing and export. +""" import csv import fnmatch -from io import BytesIO import json import os import re -from builtins import open from datetime import datetime from hashlib import md5 -from xml.dom import minidom, Node +from io import BytesIO +from xml.dom import Node, minidom -import pytz -import requests -import openpyxl from django.conf import settings from django.core.files.uploadedfile import UploadedFile from django.urls import reverse + +import openpyxl +import pytz +import requests from django_digest.test import Client as DigestClient -from six import iteritems from mock import patch +from six import iteritems from onadata.apps.logger.models import XForm from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH @@ -32,14 +36,18 @@ class TestProcess(TestBase): + """ + Test form publishing processes. + """ + loop_str = "loop_over_transport_types_frequency" frequency_str = "frequency_to_referral_facility" - ambulance_key = "%s/ambulance/%s" % (loop_str, frequency_str) - bicycle_key = "%s/bicycle/%s" % (loop_str, frequency_str) - other_key = "%s/other/%s" % (loop_str, frequency_str) - taxi_key = "%s/taxi/%s" % (loop_str, frequency_str) - transport_ambulance_key = "transport/%s" % ambulance_key - transport_bicycle_key = "transport/%s" % bicycle_key + ambulance_key = f"{loop_str}/ambulance/{frequency_str}" + bicycle_key = f"{loop_str}/bicycle/{frequency_str}" + other_key = f"{loop_str}/other/{frequency_str}" + taxi_key = f"{loop_str}/taxi/{frequency_str}" + transport_ambulance_key = "transport/{ambulance_key}" + transport_bicycle_key = "transport/{bicycle_key}" uuid_to_submission_times = { "5b2cc313-fc09-437e-8149-fcd32f695d41": "2013-02-14T15:37:21", "f3d8dc65-91a6-4d0f-9e97-802128083390": "2013-02-14T15:37:22", @@ -47,13 +55,9 @@ class TestProcess(TestBase): "9f0a1508-c3b7-4c99-be00-9b237c26bcbf": "2013-02-14T15:37:24", } - def setUp(self): - super(TestProcess, self).setUp() - - def tearDown(self): - super(TestProcess, self).tearDown() - + # pylint: disable=unused-argument def test_process(self, username=None, password=None): + """Test usage process.""" self._publish_xls_file() self._check_formlist() self._download_xform() @@ -75,6 +79,7 @@ def _update_dynamic_data(self): i.save() def test_uuid_submit(self): + """Test submission with uuid included.""" self._publish_xls_file() survey = "transport_2011-07-25_19-05-49" path = os.path.join( @@ -85,17 +90,20 @@ def test_uuid_submit(self): survey, survey + ".xml", ) - with open(path) as f: + with open(path, encoding="utf-8") as f: post_data = {"xml_submission_file": f, "uuid": self.xform.uuid} url = "/submission" + # pylint: disable=attribute-defined-outside-init self.response = self.client.post(url, post_data) def test_publish_xlsx_file(self): + """Test publishing an XLSX file.""" self._publish_xlsx_file() @patch("onadata.apps.main.forms.requests") @patch("onadata.apps.main.forms.urlopen") def test_google_url_upload(self, mock_urlopen, mock_requests): + """Test uploading an XLSForm from a Google Docs SpreadSheet URL.""" if self._internet_on(url="http://google.com"): xls_url = ( "https://docs.google.com/spreadsheet/pub?" @@ -113,35 +121,34 @@ def test_google_url_upload(self, mock_urlopen, mock_requests): "transportation.xlsx", ) - xls_file = open(path, "rb") - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response.headers = { - "content-type": ( - "application/vnd.openxmlformats-" - "officedocument.spreadsheetml.sheet" - ), - "content-disposition": ( - 'attachment; filename="transportation.' - "xlsx\"; filename*=UTF-8''transportation.xlsx" - ), - } - mock_requests.get.return_value = mock_response - mock_urlopen.return_value = xls_file - response = self.client.post( - "/%s/" % self.user.username, {"xls_url": xls_url} - ) - - mock_urlopen.assert_called_with(xls_url) - mock_requests.get.assert_called_with(xls_url) - # cleanup the resources - xls_file.close() - # make sure publishing the survey worked - self.assertEqual(response.status_code, 200) - self.assertEqual(XForm.objects.count(), pre_count + 1) + with open(path, "rb") as xls_file: + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "content-disposition": ( + 'attachment; filename="transportation.' + "xlsx\"; filename*=UTF-8''transportation.xlsx" + ), + } + mock_requests.get.return_value = mock_response + mock_urlopen.return_value = xls_file + response = self.client.post( + f"/{self.user.username}/", {"xls_url": xls_url} + ) + + mock_urlopen.assert_called_with(xls_url) + mock_requests.get.assert_called_with(xls_url) + # make sure publishing the survey worked + self.assertEqual(response.status_code, 200) + self.assertEqual(XForm.objects.count(), pre_count + 1) @patch("onadata.apps.main.forms.urlopen") def test_url_upload(self, mock_urlopen): + """Test uploading an XLSForm from a URL.""" if self._internet_on(url="http://google.com"): xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() @@ -156,25 +163,24 @@ def test_url_upload(self, mock_urlopen): "transportation.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_urlopen.return_value = xls_file - response = self.client.post( - "/%s/" % self.user.username, {"xls_url": xls_url} - ) + response = self.client.post( + f"/{self.user.username}", {"xls_url": xls_url} + ) - mock_urlopen.assert_called_with(xls_url) - # cleanup the resources - xls_file.close() + mock_urlopen.assert_called_with(xls_url) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) def test_bad_url_upload(self): + """Test uploading an XLSForm from a badly formatted URL.""" xls_url = "formhuborg/pld/forms/transportation_2011_07_25/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post("/%s/" % self.user.username, {"xls_url": xls_url}) + response = self.client.post(f"/{self.user.username}", {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count) @@ -184,10 +190,11 @@ def test_bad_url_upload(self): # containing the files you would like to test. # DO NOT CHECK IN PRIVATE XLS FILES!! def test_upload_all_xls(self): + """Test all XLSForms in online_xls folder can upload successfuly.""" root_dir = os.path.join(self.this_directory, "fixtures", "online_xls") if os.path.exists(root_dir): success = True - for root, sub_folders, filenames in os.walk(root_dir): + for root, _sub_folders, filenames in os.walk(root_dir): # ignore files that don't end in '.xlsx' for filename in fnmatch.filter(filenames, "*.xlsx"): success = self._publish_file(os.path.join(root, filename), False) @@ -195,30 +202,33 @@ def test_upload_all_xls(self): # delete it so we don't have id_string conflicts if self.xform: self.xform.delete() + # pylint: disable=attribute-defined-outside-init self.xform = None - print("finished sub-folder %s" % root) + print(f"finished sub-folder {root}") self.assertEqual(success, True) + # pylint: disable=invalid-name def test_url_upload_non_dot_xls_path(self): + """Test a non .xls URL allows XLSForm upload.""" if self._internet_on(): xls_url = "http://formhub.org/formhub_u/forms/tutorial/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post( - "/%s/" % self.user.username, {"xls_url": xls_url} - ) + response = self.client.post(f"/{self.user.username}", {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) + # pylint: disable=invalid-name def test_not_logged_in_cannot_upload(self): + """Test anonymous user cannot upload an XLSForm.""" path = os.path.join( self.this_directory, "fixtures", "transportation", "transportation.xlsx" ) - if not path.startswith("/%s/" % self.user.username): + if not path.startswith(f"/{self.user.username}"): path = os.path.join(self.this_directory, path) with open(path, "rb") as xls_file: post_data = {"xls_file": xls_file} - return self.client.post("/%s/" % self.user.username, post_data) + return self.client.post(f"/{self.user.username}", post_data) def _publish_file(self, xls_path, strict=True): """ @@ -228,15 +238,16 @@ def _publish_file(self, xls_path, strict=True): TestBase._publish_xls_file(self, xls_path) # make sure publishing the survey worked if XForm.objects.count() != pre_count + 1: - print("\nPublish Failure for file: %s" % xls_path) + print(f"\nPublish Failure for file: {xls_path}") if strict: self.assertEqual(XForm.objects.count(), pre_count + 1) else: return False + # pylint: disable=attribute-defined-outside-init self.xform = list(XForm.objects.all())[-1] return True - def _publish_xls_file(self): + def _publish_xls_file(self, path=None): xls_path = os.path.join( self.this_directory, "fixtures", "transportation", "transportation.xlsx" ) @@ -244,21 +255,17 @@ def _publish_xls_file(self): self.assertEqual(self.xform.id_string, "transportation_2011_07_25") def _check_formlist(self): - url = "/%s/formList" % self.user.username + url = f"/{self.user.username}/formList" client = DigestClient() client.set_authorization("bob", "bob") response = client.get(url) - self.download_url = "http://testserver/%s/forms/%s/form.xml" % ( - self.user.username, - self.xform.pk, + # pylint: disable=attribute-defined-outside-init + self.download_url = ( + f"http://testserver/{self.user.username}/forms/{self.xform.pk}/form.xml" ) md5_hash = md5(self.xform.xml.encode("utf-8")).hexdigest() - expected_content = """ -transportation_2011_07_25transportation_2011_07_252014111md5:%(hash)s%(download_url)s""" # noqa - expected_content = expected_content % { - "download_url": self.download_url, - "hash": md5_hash, - } + expected_content = f""" +transportation_2011_07_25transportation_2011_07_252014111md5:{md5_hash}{self.download_url}""" # noqa self.assertEqual(response.content.decode("utf-8"), expected_content) self.assertTrue(response.has_header("X-OpenRosa-Version")) self.assertTrue(response.has_header("Date")) @@ -305,13 +312,15 @@ def _check_csv_export(self): def _check_data_dictionary(self): # test to make sure the data dictionary returns the expected headers - qs = DataDictionary.objects.filter(user=self.user) - self.assertEqual(qs.count(), 1) + queryset = DataDictionary.objects.filter(user=self.user) + self.assertEqual(queryset.count(), 1) + # pylint: disable=attribute-defined-outside-init self.data_dictionary = DataDictionary.objects.all()[0] with open( os.path.join( self.this_directory, "fixtures", "transportation", "headers.json" - ) + ), + encoding="utf-8", ) as f: expected_list = json.load(f) self.assertEqual(self.data_dictionary.get_headers(), expected_list) @@ -321,7 +330,8 @@ def _check_data_dictionary(self): with open( os.path.join( self.this_directory, "fixtures", "transportation", "headers_csv.json" - ) + ), + encoding="utf-8", ) as f: expected_list = json.load(f) self.assertEqual(sorted(next(actual_csv)), sorted(expected_list)) @@ -418,6 +428,7 @@ def _check_csv_export_first_pass(self): ) self._test_csv_response(response, test_file_path) + # pylint: disable=too-many-locals def _check_csv_export_second_pass(self): url = reverse( "csv_export", @@ -494,12 +505,16 @@ def _check_csv_export_second_pass(self): }, ] - dd = DataDictionary.objects.get(pk=self.xform.pk) - additional_headers = dd._additional_headers() + ["_id", "_date_modified"] + data_dictionary = DataDictionary.objects.get(pk=self.xform.pk) + # pylint: disable=protected-access + additional_headers = data_dictionary._additional_headers() + [ + "_id", + "_date_modified", + ] for row, expected_dict in zip(actual_csv, data): test_dict = {} - d = dict(zip(headers, row)) - for (k, v) in iteritems(d): + row_dict = dict(zip(headers, row)) + for (k, v) in iteritems(row_dict): if not (v in ["n/a", "False"] or k in additional_headers): test_dict[k] = v this_list = [] @@ -511,6 +526,7 @@ def _check_csv_export_second_pass(self): self.assertEqual(test_dict, dict(this_list)) def test_xls_export_content(self): + """Test publish and export XLS content.""" self._publish_xls_file() self._make_submissions() self._update_dynamic_data() @@ -563,6 +579,7 @@ def test_405_submission(self): response = self.client.get(url) self.assertContains(response, 'Method "GET" not allowed', status_code=405) + # pylint: disable=invalid-name def test_publish_bad_xls_with_unicode_in_error(self): """ Publish an xls where the error has a unicode character @@ -575,26 +592,31 @@ def test_publish_bad_xls_with_unicode_in_error(self): ) with open(path, "rb") as xls_file: post_data = {"xls_file": xls_file} - response = self.client.post("/%s/" % self.user.username, post_data) + response = self.client.post(f"/{self.user.username}", post_data) self.assertEqual(response.status_code, 200) def test_metadata_file_hash(self): + """Test a metadata file hash is generated.""" self._publish_transportation_form() src = os.path.join( self.this_directory, "fixtures", "transportation", "screenshot.png" ) - uf = UploadedFile(file=open(src, "rb"), content_type="image/png") - count = MetaData.objects.count() - MetaData.media_upload(self.xform, uf) - # assert successful insert of new metadata record - self.assertEqual(MetaData.objects.count(), count + 1) - md = MetaData.objects.get(object_id=self.xform.id, data_value="screenshot.png") - # assert checksum string has been generated, hash length > 1 - self.assertTrue(len(md.hash) > 16) + with open(src, "rb") as screenshot_file: + upload_file = UploadedFile(file=screenshot_file, content_type="image/png") + count = MetaData.objects.count() + MetaData.media_upload(self.xform, upload_file) + # assert successful insert of new metadata record + self.assertEqual(MetaData.objects.count(), count + 1) + metadata = MetaData.objects.get( + object_id=self.xform.id, data_value="screenshot.png" + ) + # assert checksum string has been generated, hash length > 1 + self.assertTrue(len(metadata.hash) > 16) + # pylint: disable=invalid-name def test_uuid_injection_in_cascading_select(self): """ - Uuid is injected in the right instance for forms with cascading select + UUID is injected in the right instance for forms with cascading select """ pre_count = XForm.objects.count() xls_path = os.path.join( @@ -603,7 +625,6 @@ def test_uuid_injection_in_cascading_select(self): "cascading_selects", "new_cascading_select.xlsx", ) - file_name, file_ext = os.path.splitext(os.path.split(xls_path)[1]) TestBase._publish_xls_file(self, xls_path) post_count = XForm.objects.count() self.assertEqual(post_count, pre_count + 1) @@ -652,10 +673,11 @@ def test_uuid_injection_in_cascading_select(self): self.assertEqual(len(calculate_bind_nodes), 1) calculate_bind_node = calculate_bind_nodes[0] self.assertEqual( - calculate_bind_node.getAttribute("calculate"), "'%s'" % xform.uuid + calculate_bind_node.getAttribute("calculate"), f"'{xform.uuid}'" ) def test_csv_publishing(self): + """Test publishing a CSV XLSForm.""" csv_text = "\n".join( [ "survey,,", @@ -667,20 +689,23 @@ def test_csv_publishing(self): url = reverse("user_profile", kwargs={"username": self.user.username}) num_xforms = XForm.objects.count() params = {"text_xls_form": csv_text} - self.response = self.client.post(url, params) + self.client.post(url, params) self.assertEqual(XForm.objects.count(), num_xforms + 1) + # pylint: disable=invalid-name def test_truncate_xform_title_to_255(self): + """Test the XLSForm title is truncated at 255 characters.""" self._publish_transportation_form() title = "a" * (XFORM_TITLE_LENGTH + 1) groups = re.match( r"(.+)([^<]+)(.*)", self.xform.xml, re.DOTALL ).groups() - self.xform.xml = "{0}{1}{2}".format(groups[0], title, groups[2]) + self.xform.xml = f"{groups[0]}{title}{groups[2]}" self.xform.title = title self.xform.save() self.assertEqual(self.xform.title, "a" * XFORM_TITLE_LENGTH) + # pylint: disable=invalid-name def test_multiple_submissions_by_different_users(self): """ Two users publishing the same form breaks the CSV export. diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index a6e7a1fd79..8c1e96b729 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -59,14 +59,11 @@ def report_exception(subject, info, exc_info=None): testing sends email to mail_admins. """ # Add hostname to subject mail - subject = "{0} - {1}".format(subject, settings.HOSTNAME) + subject = f"{subject} - {settings.HOSTNAME}" if exc_info: cls, err = exc_info[:2] - message = _("Exception in request:" " %(class)s: %(error)s") % { - "class": cls.__name__, - "error": err, - } + message = _(f"Exception in request: {cls.__name__}: {err}") message += "".join(traceback.format_exception(*exc_info)) # send to sentry @@ -75,11 +72,11 @@ def report_exception(subject, info, exc_info=None): except Exception: # pylint: disable=broad-except logging.exception(_("Sending to Sentry failed.")) else: - message = "%s" % info + message = f"{info}" if settings.DEBUG or settings.TESTING_MODE: - sys.stdout.write("Subject: %s\n" % subject) - sys.stdout.write("Message: %s\n" % message) + sys.stdout.write(f"Subject: {subject}\n") + sys.stdout.write(f"Message: {message}\n") else: mail_admins(subject=subject, message=message) @@ -117,8 +114,7 @@ def get_response_content(response, decode=True): if decode: return contents.decode("utf-8") - else: - return contents + return contents def json_stream(data, json_string): @@ -181,6 +177,7 @@ def function_retry(self, *args, **kwargs): return result # Last ditch effort run against master database if len(getattr(settings, "SLAVE_DATABASES", [])): + # pylint: disable=import-outside-toplevel from multidb.pinning import use_master with use_master: @@ -209,7 +206,9 @@ def merge_dicts(*dict_args): def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" - class K(object): + class ComparatorClass: + """A class that implements comparison methods.""" + def __init__(self, obj, *args): self.obj = obj @@ -231,4 +230,4 @@ def __ge__(self, other): def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 - return K + return ComparatorClass diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 28434b9aa3..dfe02ff0c4 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -252,30 +252,31 @@ def generate_enketo_form_defaults(xform, **kwargs): def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file - with NamedTemporaryFile() as tmp: - with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: - for attachment in attachments: - default_storage = get_storage_class()() - filename = attachment.media_file.name - - if default_storage.exists(filename): - try: - with default_storage.open(filename) as f: - if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: - report_exception( - "Create attachment zip exception", - ( - "File is greater than " - f"{settings.ZIP_REPORT_ATTACHMENT_LIMIT} bytes" - ), - ) - break - z.writestr(attachment.media_file.name, f.read()) - except IOError as e: - report_exception("Create attachment zip exception", e) - break - - return tmp + # pylint: disable=consider-using-with + tmp = NamedTemporaryFile() + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: + for attachment in attachments: + default_storage = get_storage_class()() + filename = attachment.media_file.name + + if default_storage.exists(filename): + try: + with default_storage.open(filename) as f: + if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: + report_exception( + "Create attachment zip exception", + ( + "File is greater than " + f"{settings.ZIP_REPORT_ATTACHMENT_LIMIT} bytes" + ), + ) + break + z.writestr(attachment.media_file.name, f.read()) + except IOError as e: + report_exception("Create attachment zip exception", e) + break + + return tmp def get_form(kwargs): From 122b72fb8ec642bce43835ce6d4aae614af4eaa3 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 14:21:19 +0300 Subject: [PATCH 094/234] batch: cleanup --- .../tests/viewsets/test_abstract_viewset.py | 7 ++- .../management/commands/import_instances.py | 9 ++-- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/main/tests/test_process.py | 4 +- .../libs/serializers/metadata_serializer.py | 49 +++++++++---------- 5 files changed, 38 insertions(+), 33 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 93f1e6a1f2..13ec6a3641 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -106,6 +106,7 @@ def setUp(self): self.maxDiff = None def user_profile_data(self): + """Returns the user profile python object.""" return { "id": self.user.pk, "url": "http://testserver/api/v1/profiles/bob", @@ -493,8 +494,10 @@ def _create_dataview(self, data=None, project=None, xform=None): "xform": f"http://testserver/api/v1/forms/{xform.pk}", "project": f"http://testserver/api/v1/projects/{project.pk}", "columns": '["name", "age", "gender"]', - "query": '[{"column":"age","filter":">","value":"20"},' - '{"column":"age","filter":"<","value":"50"}]', + "query": ( + '[{"column":"age","filter":">","value":"20"},' + '{"column":"age","filter":"<","value":"50"}]' + ), } request = self.factory.post("/", data=data, **self.extra) response = view(request) diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index f60d8191fd..ec61c47b78 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=5 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=5 +# -*- coding: utf-8 -*- """ import_instances - import ODK instances from a zipped file. """ @@ -41,6 +42,7 @@ def _log_import(self, results): % {"total": total_count, "imported": success_count, "errors": errors} ) + # pylint: disable=unused-argument def handle(self, *args, **kwargs): if len(args) < 2: raise CommandError(_("Usage: username file/path.")) @@ -72,10 +74,11 @@ def handle(self, *args, **kwargs): self._log_import(results) for file in files: filepath = os.path.join(path, file) - if ( + is_zip_file = ( os.path.isfile(filepath) and os.path.splitext(filepath)[1].lower() == ".zip" - ): + ) + if is_zip_file: self.stdout.write(_(f"Importing from zip at {filepath}..\n")) results = import_instances_from_zip(filepath, user) self._log_import(results) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 5f3dc6fac6..8d7fb9867b 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -352,7 +352,7 @@ def point(self): """Returns the Point of the first geom if it is a collection.""" geom_collection = self.geom - if geom_collection and isinstance(geom_collection, list): + if geom_collection and geom_collection.num_points: return geom_collection[0] return self.geom diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index fb1e772b94..37ebd816b8 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -46,8 +46,8 @@ class TestProcess(TestBase): bicycle_key = f"{loop_str}/bicycle/{frequency_str}" other_key = f"{loop_str}/other/{frequency_str}" taxi_key = f"{loop_str}/taxi/{frequency_str}" - transport_ambulance_key = "transport/{ambulance_key}" - transport_bicycle_key = "transport/{bicycle_key}" + transport_ambulance_key = f"transport/{ambulance_key}" + transport_bicycle_key = f"transport/{bicycle_key}" uuid_to_submission_times = { "5b2cc313-fc09-437e-8149-fcd32f695d41": "2013-02-14T15:37:21", "f3d8dc65-91a6-4d0f-9e97-802128083390": "2013-02-14T15:37:22", diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 967dbc56bb..605f5d36c9 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -75,7 +75,7 @@ def get_linked_object(parts): obj_pk = parts[1] try: obj_pk = int(obj_pk) - except ValueError: + except ValueError as e: raise serializers.ValidationError( { "data_value": _( @@ -83,11 +83,12 @@ def get_linked_object(parts): % {"type": obj_type, "id": obj_pk} ) } - ) + ) from e else: model = DataView if obj_type == DATAVIEW_TAG else XForm return get_object_or_404(model, pk=obj_pk) + return None class MetaDataSerializer(serializers.HyperlinkedModelSerializer): @@ -95,7 +96,7 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): MetaData HyperlinkedModelSerializer """ - id = serializers.ReadOnlyField() # pylint: disable=C0103 + id = serializers.ReadOnlyField() # pylint: disable=invalid-name xform = XFormRelatedField(queryset=XForm.objects.all(), required=False) project = ProjectRelatedField(queryset=Project.objects.all(), required=False) instance = InstanceRelatedField(queryset=Instance.objects.all(), required=False) @@ -138,7 +139,7 @@ def get_media_url(self, obj): and getattr(obj.data_file, "url") ): return obj.data_file.url - elif obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: + if obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: kwargs = { "kwargs": { "pk": obj.content_object.pk, @@ -150,7 +151,9 @@ def get_media_url(self, obj): } return reverse("xform-media", **kwargs) + return None + # pylint: disable=too-many-branches def validate(self, attrs): """ Validate url if we are adding a media uri instead of a media file @@ -177,19 +180,14 @@ def validate(self, attrs): ) if data_content_type not in allowed_types: raise serializers.ValidationError( - { - "data_file": _( - "Unsupported media file type %s" % data_content_type - ) - } + {"data_file": _(f"Unsupported media file type {data_content_type}")} ) - else: - attrs["data_file_type"] = data_content_type + attrs["data_file_type"] = data_content_type if data_type == "media" and data_file is None: try: URLValidator()(value) - except ValidationError: + except ValidationError as e: parts = value.split() if len(parts) < 3: raise serializers.ValidationError( @@ -216,20 +214,20 @@ def validate(self, attrs): "User has no permission to " "the dataview." ) } - ) + ) from e else: raise serializers.ValidationError( - {"data_value": _("Invalid url '%s'." % value)} - ) + {"data_value": _(f"Invalid url '{value}'.")} + ) from e else: # check if we have a value for the filename. if not os.path.basename(urlparse(value).path): raise serializers.ValidationError( { "data_value": _( - "Cannot get filename from URL %s. URL should " + f"Cannot get filename from URL {value}. URL should " "include the filename e.g " - "http://example.com/data.csv" % value + "http://example.com/data.csv" ) } ) @@ -243,7 +241,7 @@ def validate(self, attrs): return attrs - # pylint: disable=R0201 + # pylint: disable=no-self-use def get_content_object(self, validated_data): """ Returns the validated 'xform' or 'project' or 'instance' ids being @@ -256,6 +254,7 @@ def get_content_object(self, validated_data): or validated_data.get("project") or validated_data.get("instance") ) + return None def create(self, validated_data): data_type = validated_data.get("data_type") @@ -289,10 +288,9 @@ def create(self, validated_data): # ensure only one submission_review metadata exists per form if MetaData.submission_review(content_object): raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) - else: - metadata = MetaData.submission_review( - content_object, data_value=data_value - ) + metadata = MetaData.submission_review( + content_object, data_value=data_value + ) elif data_type == IMPORTED_VIA_CSV_BY: metadata = MetaData.instance_csv_imported_by( content_object, data_value=data_value @@ -308,11 +306,12 @@ def create(self, validated_data): ) return metadata - except IntegrityError: - raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) + except IntegrityError as e: + raise serializers.ValidationError(_(UNIQUE_TOGETHER_ERROR)) from e + return None def update(self, instance, validated_data): - instance = super(MetaDataSerializer, self).update(instance, validated_data) + instance = super().update(instance, validated_data) if instance.data_type == XFORM_META_PERMS: update_role_by_meta_xform_perms(instance.content_object) From 20f1d3b5f7dc4c1cdf0f403d7973de35d0e53573 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 15:18:54 +0300 Subject: [PATCH 095/234] batch: cleanup --- .../apps/api/viewsets/briefcase_viewset.py | 68 +++++++++++-------- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/main/tests/test_process.py | 11 +-- onadata/libs/utils/briefcase_client.py | 1 + 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 09d4a29e50..6770d81406 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -1,26 +1,25 @@ -import six - +# -*- coding: utf-8 -*- +""" +The /briefcase API implementation. +""" from xml.dom import NotFoundErr from django.conf import settings +from django.contrib.auth import get_user_model from django.core.files import File from django.core.validators import ValidationError -from django.contrib.auth import get_user_model from django.http import Http404 from django.utils.translation import ugettext as _ -from rest_framework import exceptions -from rest_framework import mixins -from rest_framework import status -from rest_framework import viewsets -from rest_framework import permissions +import six +from rest_framework import exceptions, mixins, permissions, status, viewsets from rest_framework.decorators import action from rest_framework.generics import get_object_or_404 from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response -from onadata.apps.api.tools import get_media_file_response from onadata.apps.api.permissions import ViewDjangoObjectPermissions +from onadata.apps.api.tools import get_media_file_response from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm @@ -31,13 +30,13 @@ from onadata.libs.authentication import DigestAuthentication from onadata.libs.mixins.openrosa_headers_mixin import get_openrosa_headers from onadata.libs.renderers.renderers import TemplateXMLRenderer -from onadata.libs.serializers.xform_serializer import XFormListSerializer -from onadata.libs.serializers.xform_serializer import XFormManifestSerializer -from onadata.libs.utils.logger_tools import publish_form -from onadata.libs.utils.logger_tools import PublishXForm +from onadata.libs.serializers.xform_serializer import ( + XFormListSerializer, + XFormManifestSerializer, +) +from onadata.libs.utils.logger_tools import PublishXForm, publish_form from onadata.libs.utils.viewer_tools import get_form - User = get_user_model() @@ -57,11 +56,11 @@ def _extract_uuid(text): return text -def _extract_id_string(formId): - if isinstance(formId, six.string_types): - return formId[0 : formId.find("[")] +def _extract_id_string(id_string): + if isinstance(id_string, six.string_types): + return id_string[0 : id_string.find("[")] - return formId + return id_string def _parse_int(num): @@ -71,6 +70,7 @@ def _parse_int(num): return None +# pylint: disable=too-many-ancestors class BriefcaseViewset( mixins.CreateModelMixin, mixins.RetrieveModelMixin, @@ -125,12 +125,12 @@ def filter_queryset(self, queryset): else: queryset = super().filter_queryset(queryset) - formId = self.request.GET.get("formId", "") + id_string = self.request.GET.get("formId", "") - if formId.find("[") != -1: - formId = _extract_id_string(formId) + if id_string.find("[") != -1: + id_string = _extract_id_string(id_string) - xform_kwargs = {"queryset": queryset, "id_string__iexact": formId} + xform_kwargs = {"queryset": queryset, "id_string__iexact": id_string} if username: xform_kwargs["user__username__iexact"] = username xform = get_form(xform_kwargs) @@ -158,17 +158,19 @@ def filter_queryset(self, queryset): # and removes the need to perform a count on the database. instance_count = len(instances) + # pylint: disable=attribute-defined-outside-init if instance_count > 0: last_instance = instances[instance_count - 1] - self.resumptionCursor = last_instance.get("pk") + self.resumption_cursor = last_instance.get("pk") elif instance_count == 0 and cursor: - self.resumptionCursor = cursor + self.resumption_cursor = cursor else: - self.resumptionCursor = 0 + self.resumption_cursor = 0 return instances def create(self, request, *args, **kwargs): + """Accepts an XForm XML and publishes it as a form.""" if request.method.upper() == "HEAD": return Response( status=status.HTTP_204_NO_CONTENT, @@ -216,11 +218,13 @@ def create(self, request, *args, **kwargs): ) def list(self, request, *args, **kwargs): + """Returns a list of submissions with reference submission download.""" + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) data = { "instances": self.object_list, - "resumptionCursor": self.resumptionCursor, + "resumptionCursor": self.resumption_cursor, } return Response( @@ -230,6 +234,8 @@ def list(self, request, *args, **kwargs): ) def retrieve(self, request, *args, **kwargs): + """Returns a single submission XML for download.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() xml_obj = clean_and_parse_xml(self.object.xml) @@ -260,6 +266,8 @@ def retrieve(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def manifest(self, request, *args, **kwargs): + """Returns list of media content.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() object_list = MetaData.objects.filter( data_type="media", object_id=self.object.id @@ -273,14 +281,16 @@ def manifest(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def media(self, request, *args, **kwargs): + """Returns a single media content.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - pk = kwargs.get("metadata") + metadata_pk = kwargs.get("metadata") - if not pk: + if not metadata_pk: raise Http404() meta_obj = get_object_or_404( - MetaData, data_type="media", xform=self.object, pk=pk + MetaData, data_type="media", xform=self.object, pk=metadata_pk ) return get_media_file_response(meta_obj) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 8d7fb9867b..e615087da9 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -413,7 +413,7 @@ def get_full_dict(self, load_existing=True): doc = self.get_dict() # pylint: disable=no-member if self.id: - geopoint = ([self.point.y, self.point.x] if self.point else [None, None],) + geopoint = [self.point.y, self.point.x] if self.point else [None, None] doc.update( { UUID: self.uuid, diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 37ebd816b8..86e03e184f 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -163,24 +163,25 @@ def test_url_upload(self, mock_urlopen): "transportation.xlsx", ) + # pylint: disable=consider-using-with with open(path, "rb") as xls_file: mock_urlopen.return_value = xls_file response = self.client.post( - f"/{self.user.username}", {"xls_url": xls_url} + f"/{self.user.username}/", {"xls_url": xls_url} ) mock_urlopen.assert_called_with(xls_url) - # make sure publishing the survey worked - self.assertEqual(response.status_code, 200) - self.assertEqual(XForm.objects.count(), pre_count + 1) + # make sure publishing the survey worked + self.assertEqual(response.status_code, 200) + self.assertEqual(XForm.objects.count(), pre_count + 1) def test_bad_url_upload(self): """Test uploading an XLSForm from a badly formatted URL.""" xls_url = "formhuborg/pld/forms/transportation_2011_07_25/form.xlsx" pre_count = XForm.objects.count() - response = self.client.post(f"/{self.user.username}", {"xls_url": xls_url}) + response = self.client.post(f"/{self.user.username}/", {"xls_url": xls_url}) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count) diff --git a/onadata/libs/utils/briefcase_client.py b/onadata/libs/utils/briefcase_client.py index a928a7c7b2..5a43c8ad59 100644 --- a/onadata/libs/utils/briefcase_client.py +++ b/onadata/libs/utils/briefcase_client.py @@ -74,6 +74,7 @@ def _get_instances_uuids(xml_doc): return uuids +# pylint: disable=too-many-instance-attributes class BriefcaseClient: """ODK BriefcaseClient class""" From 2d22c6b29cd24101cf25fae8e7c7e1c1f231bedb Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 16:20:30 +0300 Subject: [PATCH 096/234] Fixes --- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/main/tests/test_process.py | 2 +- onadata/libs/utils/viewer_tools.py | 19 ++++++++++--------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index e615087da9..bd59e70c0e 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -164,7 +164,7 @@ def get_id_string_from_xml_str(xml_str): root_node = xml_obj.documentElement id_string = root_node.getAttribute("id") - if id_string: + if not id_string: # may be hidden in submission/data/id_string elems = root_node.getElementsByTagName("data") diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 86e03e184f..0c48d651e8 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -593,7 +593,7 @@ def test_publish_bad_xls_with_unicode_in_error(self): ) with open(path, "rb") as xls_file: post_data = {"xls_file": xls_file} - response = self.client.post(f"/{self.user.username}", post_data) + response = self.client.post(f"/{self.user.username}/", post_data) self.assertEqual(response.status_code, 200) def test_metadata_file_hash(self): diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index dfe02ff0c4..ce3e6e396c 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -127,15 +127,16 @@ def django_file(path, field_name, content_type): # adapted from here: http://groups.google.com/group/django-users/browse_th\ # read/thread/834f988876ff3c45/ - with open(path, "rb") as file_object: - return InMemoryUploadedFile( - file=file_object, - field_name=field_name, - name=file_object.name, - content_type=content_type, - size=os.path.getsize(path), - charset=None, - ) + # pylint: disable=consider-using-with + file_object = open(path, "rb") + return InMemoryUploadedFile( + file=file_object, + field_name=field_name, + name=file_object.name, + content_type=content_type, + size=os.path.getsize(path), + charset=None, + ) def export_def_from_filename(filename): From ebbe2672fd496cd66d1f1ad7b6e6b4939bdd90d9 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 16:45:22 +0300 Subject: [PATCH 097/234] onadata.libs.utils.audit is not in use --- onadata/libs/utils/audit.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 onadata/libs/utils/audit.py diff --git a/onadata/libs/utils/audit.py b/onadata/libs/utils/audit.py deleted file mode 100644 index 706750dd2c..0000000000 --- a/onadata/libs/utils/audit.py +++ /dev/null @@ -1 +0,0 @@ -HOME_ACCESSED = "home-accessed" From a8cc0d56d1ca39a0701c58cab39b06be6b38fedd Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 20:54:33 +0300 Subject: [PATCH 098/234] batch: cleanup --- onadata/apps/api/tools.py | 70 +++-- onadata/apps/api/viewsets/data_viewset.py | 7 + onadata/apps/api/viewsets/project_viewset.py | 5 + onadata/apps/main/models/audit.py | 4 +- onadata/apps/main/models/meta_data.py | 24 +- onadata/apps/main/tests/test_base.py | 5 +- onadata/libs/serializers/widget_serializer.py | 12 +- onadata/libs/utils/api_export_tools.py | 48 ++- onadata/libs/utils/csv_builder.py | 280 +++++++++--------- onadata/libs/utils/osm.py | 17 +- 10 files changed, 249 insertions(+), 223 deletions(-) diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 3fd81192de..1092998fb7 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -8,7 +8,8 @@ from django import forms from django.conf import settings -from django.contrib.auth.models import Permission, User +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType from django.contrib.sites.models import Site from django.core.files.storage import get_storage_class @@ -19,16 +20,16 @@ from django.db.utils import IntegrityError from django.http import HttpResponseNotFound, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ from django.utils.module_loading import import_string -from six import iteritems -from guardian.shortcuts import assign_perm, get_perms_for_model, remove_perm -from guardian.shortcuts import get_perms +from django.utils.translation import ugettext as _ + +from guardian.shortcuts import assign_perm, get_perms, get_perms_for_model, remove_perm from kombu.exceptions import OperationalError +from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import exceptions +from six import iteritems from taggit.forms import TagField -from multidb.pinning import use_master from onadata.apps.api.models.organization_profile import ( OrganizationProfile, @@ -81,6 +82,9 @@ DECIMAL_PRECISION = 2 +# pylint: disable=invalid-name +User = get_user_model() + def _get_first_last_names(name): name_split = name.split() @@ -157,8 +161,8 @@ def create_organization_object(org_name, creator, attrs=None): new_user.save() try: registration_profile = RegistrationProfile.objects.create_profile(new_user) - except IntegrityError: - raise ValidationError(_("%s already exists" % org_name)) + except IntegrityError as e: + raise ValidationError(_(f"{org_name} already exists")) from e if email: site = Site.objects.get(pk=settings.SITE_ID) registration_profile.send_activation_email(site) @@ -203,7 +207,7 @@ def get_organization_members_team(organization): create members team if it does not exist and add organization owner to the members team""" try: - team = Team.objects.get(name="%s#%s" % (organization.user.username, MEMBERS)) + team = Team.objects.get(name=f"{organization.user.username}#{MEMBERS}") except Team.DoesNotExist: team = create_organization_team(organization, MEMBERS) add_user_to_team(team, organization.user) @@ -222,8 +226,6 @@ def get_or_create_organization_owners_team(org): try: team = Team.objects.get(name=team_name, organization=org.user) except Team.DoesNotExist: - from multidb.pinning import use_master # pylint: disable=import-error - with use_master: queryset = Team.objects.filter(name=team_name, organization=org.user) if queryset.count() > 0: @@ -383,14 +385,13 @@ def do_publish_xlsform(user, post, files, owner, id_string=None, project=None): ) if not ManagerRole.user_has_role(user, xform): raise exceptions.PermissionDenied( - _("{} has no manager/owner role to the form {}".format(user, xform)) + _(f"{user} has no manager/owner role to the form {xform}") ) elif not user.has_perm("can_add_xform", owner.profile): raise exceptions.PermissionDenied( detail=_( - "User %(user)s has no permission to add xforms to " - "account %(account)s" - % {"user": user.username, "account": owner.username} + f"User {user.username} has no permission to add xforms to " + f"account {owner.username}" ) ) @@ -400,7 +401,7 @@ def set_form(): """ if project: - args = (post and dict(list(iteritems(post)))) or {} + args = dict(list(iteritems(post))) if post else {} args["project"] = project.pk else: args = post @@ -448,18 +449,14 @@ def id_string_exists_in_account(): if "formid" in request.data: xform = get_object_or_404(XForm, pk=request.data.get("formid")) - safe_delete("{}{}".format(PROJ_OWNER_CACHE, xform.project.pk)) - safe_delete("{}{}".format(PROJ_FORMS_CACHE, xform.project.pk)) - safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, xform.project.pk)) - safe_delete("{}{}".format(PROJ_NUM_DATASET_CACHE, xform.project.pk)) - safe_delete("{}{}".format(PROJ_SUB_DATE_CACHE, xform.project.pk)) + safe_delete(f"{PROJ_OWNER_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_FORMS_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_BASE_FORMS_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_NUM_DATASET_CACHE}{xform.project.pk}") + safe_delete(f"{PROJ_SUB_DATE_CACHE}{xform.project.pk}") if not ManagerRole.user_has_role(request.user, xform): raise exceptions.PermissionDenied( - _( - "{} has no manager/owner role to the form {}".format( - request.user, xform - ) - ) + _(f"{request.user} has no manager/owner role to the form {xform}") ) msg = "Form with the same id_string already exists in this account" @@ -474,8 +471,8 @@ def id_string_exists_in_account(): try: with transaction.atomic(): xform.save() - except IntegrityError: - raise exceptions.ParseError(_(msg)) + except IntegrityError as e: + raise exceptions.ParseError(_(msg)) from e else: # First assign permissions to the person who uploaded the form OwnerRole.add(request.user, xform) @@ -709,9 +706,9 @@ def generate_tmp_path(uploaded_csv_file): """ if isinstance(uploaded_csv_file, InMemoryUploadedFile): uploaded_csv_file.open() - tmp_file = tempfile.NamedTemporaryFile(delete=False) - tmp_file.write(uploaded_csv_file.read()) - tmp_path = tmp_file.name + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + tmp_file.write(uploaded_csv_file.read()) + tmp_path = tmp_file.name uploaded_csv_file.close() else: tmp_path = uploaded_csv_file.temporary_file_path() @@ -756,10 +753,10 @@ def get_xform_users(xform): "user": user.username, } - for k in data: - data[k]["permissions"].sort() - data[k]["role"] = get_role(data[k]["permissions"], xform) - del data[k]["permissions"] + for value in data.values(): + value["permissions"].sort() + value["role"] = get_role(value["permissions"], xform) + del value["permissions"] return data @@ -772,7 +769,7 @@ def get_team_members(org_username): """ members = [] try: - team = Team.objects.get(name="{}#{}".format(org_username, MEMBERS)) + team = Team.objects.get(name=f"{org_username}#{MEMBERS}") except Team.DoesNotExist: pass else: @@ -812,6 +809,7 @@ def update_role_by_meta_xform_perms(xform): def replace_attachment_name_with_url(data): + """Replaces the attachment filename with a URL in ``data`` object.""" site_url = Site.objects.get_current().domain for record in data: diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index b6d2bf2bc4..2ddf538979 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -73,6 +73,7 @@ settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 ) +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() @@ -145,6 +146,7 @@ class DataViewSet( queryset = XForm.objects.filter(deleted_at__isnull=True) def get_serializer_class(self): + """Returns appropriate serializer class based on context.""" pk_lookup, dataid_lookup = self.lookup_fields form_pk = self.kwargs.get(pk_lookup) dataid = self.kwargs.get(dataid_lookup) @@ -173,6 +175,7 @@ def get_serializer_class(self): # pylint: disable=unused-argument def get_object(self, queryset=None): + """Returns the appropriate object based on context.""" obj = super().get_object() pk_lookup, dataid_lookup = self.lookup_fields form_pk = self.kwargs.get(pk_lookup) @@ -218,6 +221,7 @@ def _filtered_or_shared_queryset(self, queryset, form_pk): # pylint: disable=unused-argument def filter_queryset(self, queryset, view=None): + """Returns and filters queryset based on context and query params.""" queryset = super().filter_queryset(queryset.only("id", "shared")) form_pk = self.kwargs.get(self.lookup_field) @@ -332,6 +336,7 @@ def enketo(self, request, *args, **kwargs): return Response(data=data) def destroy(self, request, *args, **kwargs): + """Soft deletes submissions data.""" instance_ids = request.data.get("instance_ids") delete_all_submissions = strtobool(request.data.get("delete_all", "False")) # pylint: disable=attribute-defined-outside-init @@ -406,6 +411,7 @@ def destroy(self, request, *args, **kwargs): return Response(status=status.HTTP_204_NO_CONTENT) def retrieve(self, request, *args, **kwargs): + """Returns API data for the targeted object.""" _data_id, _format = get_data_and_form(kwargs) # pylint: disable=attribute-defined-outside-init self.object = instance = self.get_object() @@ -506,6 +512,7 @@ def _set_pagination_headers( # pylint: disable=too-many-locals,too-many-branches,too-many-statements def list(self, request, *args, **kwargs): + """Returns list of data API endpoints for different forms.""" fields = request.GET.get("fields") query = request.GET.get("query", {}) sort = request.GET.get("sort") diff --git a/onadata/apps/api/viewsets/project_viewset.py b/onadata/apps/api/viewsets/project_viewset.py index 4439b793ca..3c0da0d1c0 100644 --- a/onadata/apps/api/viewsets/project_viewset.py +++ b/onadata/apps/api/viewsets/project_viewset.py @@ -61,6 +61,7 @@ class ProjectViewSet( List, Retrieve, Update, Create Project and Project Forms. """ + # pylint: disable=no-member queryset = Project.objects.filter(deleted_at__isnull=True).select_related() serializer_class = ProjectSerializer lookup_field = "pk" @@ -69,11 +70,13 @@ class ProjectViewSet( filter_backends = (AnonUserProjectFilter, ProjectOwnerFilter, TagFilter) def get_serializer_class(self): + """Return BaseProjectSerializer class when listing projects.""" if self.action == "list": return BaseProjectSerializer return super().get_serializer_class() def get_queryset(self): + """Use 'prepared' prefetched queryset for GET requests.""" if self.request.method.upper() in ["GET", "OPTIONS"]: self.queryset = Project.prefetched.filter( deleted_at__isnull=True, organization__is_active=True @@ -82,6 +85,7 @@ def get_queryset(self): return super().get_queryset() def update(self, request, *args, **kwargs): + """Updates project properties and set's cache with the updated records.""" project_id = kwargs.get("pk") response = super().update(request, *args, **kwargs) cache.set(f"{PROJ_OWNER_CACHE}{project_id}", response.data) @@ -206,6 +210,7 @@ def star(self, request, *args, **kwargs): return Response(status=status.HTTP_204_NO_CONTENT) def destroy(self, request, *args, **kwargs): + """ "Soft deletes a project""" project = self.get_object() user = request.user project.soft_delete(user) diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 27237aed3a..3e11f55834 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -17,6 +17,7 @@ class Audit(models.Model): Audit model - persists audit logs. """ + # pylint: disable=no-member json = models.JSONField() class Meta: @@ -52,6 +53,7 @@ def query_iterator(cls, sql, fields=None, params=None, count=False): # cursor seems to stringify dicts # added workaround to parse stringified dicts to json def parse_json(data): + """Helper function to return a JSON string ``data`` as a python object.""" try: return json.loads(data) except ValueError: @@ -154,7 +156,7 @@ def query_data( sql, params = records.query.sql_with_params() - if isinstance(sort, six.string_types) and len(sort) > 0: + if isinstance(sort, six.string_types) and sort: direction = "DESC" if sort.startswith("-") else "ASC" sort = sort[1:] if sort.startswith("-") else sort sql = f"{sql} ORDER BY json->>%s {direction}" diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index 8b1261b14d..5f2dbd1aa0 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -8,7 +8,7 @@ import mimetypes import os from contextlib import closing -from hashlib import md5 +import hashlib from django.conf import settings from django.contrib.contenttypes.fields import GenericForeignKey @@ -31,23 +31,19 @@ XFORM_META_PERMS, ) +ANONYMOUS_USERNAME = "anonymous" CHUNK_SIZE = 1024 INSTANCE_MODEL_NAME = "instance" PROJECT_MODEL_NAME = "project" XFORM_MODEL_NAME = "xform" -urlvalidate = URLValidator() - -ANONYMOUS_USERNAME = "anonymous" - - def is_valid_url(uri): """ Validates a URI. """ try: - urlvalidate(uri) + URLValidator()(uri) except ValidationError: return False @@ -58,12 +54,10 @@ def upload_to(instance, filename): """ Returns the upload path for given ``filename``. """ + is_instance_model = instance.content_type.model == INSTANCE_MODEL_NAME username = None - if ( - instance.content_object.user is None - and instance.content_type.model == INSTANCE_MODEL_NAME - ): + if instance.content_object.user is None and is_instance_model: username = instance.content_object.xform.user.username else: username = instance.content_object.user.username @@ -107,6 +101,7 @@ def unique_type_for_form(content_object, data_type, data_value=None, data_file=N content_type = ContentType.objects.get_for_model(content_object) if data_value is None and data_file is None: + # pylint: disable=no-member result = MetaData.objects.filter( object_id=content_object.id, content_type=content_type, data_type=data_type ).first() @@ -145,6 +140,7 @@ def create_media(media): data_file = NamedTemporaryFile() content_type = mimetypes.guess_type(filename) with closing(requests.get(media.data_value, stream=True)) as resp: + # pylint: disable=no-member for chunk in resp.iter_content(chunk_size=CHUNK_SIZE): if chunk: data_file.write(chunk) @@ -206,6 +202,7 @@ class Meta: app_label = "main" unique_together = ("object_id", "data_type", "data_value", "content_type") + # pylint: disable=arguments-differ def save(self, *args, **kwargs): self._set_hash() super().save(*args, **kwargs) @@ -237,7 +234,10 @@ def _set_hash(self): except IOError: return "" else: - self.file_hash = f"md5:{md5(self.data_file.read()).hexdigest()}" + file_hash = hashlib.new( + "md5", self.data_file.read(), usedforsecurity=False + ).hexdigest() + self.file_hash = f"md5:{file_hash}" return self.file_hash diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index 9431dd9a60..deee090673 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -19,11 +19,12 @@ from django.test.client import Client from django.utils import timezone +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen + from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth from rest_framework.test import APIRequestFactory -from six.moves.urllib.error import URLError -from six.moves.urllib.request import urlopen from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment, Instance, XForm diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index e82d48206a..516849f755 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -51,6 +51,7 @@ def _setup_field(self, view_name): self.queryset = DataView.objects.all() def to_representation(self, value): + """Set's the self.view_name based on the type of ``value``.""" if isinstance(value, XForm): # pylint: disable=attribute-defined-outside-init self.view_name = "xform-detail" @@ -62,10 +63,11 @@ def to_representation(self, value): self._setup_field(self.view_name) - # pylint: disable=bad-super-call,super-with-arguments + # pylint: disable=bad-super-call return super(GenericRelatedField, self).to_representation(value) def to_internal_value(self, data): + """Verifies that ``data`` is a valid URL.""" try: http_prefix = data.startswith(("http:", "https:")) except AttributeError: @@ -155,6 +157,7 @@ def get_data(self, obj): return data def validate(self, attrs): + """Validates that column exists in the XForm.""" column = attrs.get("column") # Get the form @@ -194,11 +197,8 @@ def validate_content_object(self, value): profile = value.project.organization.profile # Shared or an admin in the organization - if ( - request.user not in users - and not is_organization(profile) - and not OwnerRole.user_has_role(request.user, profile) - ): + is_owner = OwnerRole.user_has_role(request.user, profile) + if request.user not in users and not is_organization(profile) and not is_owner: raise serializers.ValidationError( _("You don't have permission to the Project.") ) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 30d4f2f09a..600dd55043 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -23,7 +23,6 @@ TokenRevokeError, ) from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage -from requests import ConnectionError from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse @@ -108,10 +107,7 @@ def _get_export_type(export_type): export_type = EXPORT_EXT[export_type] else: raise exceptions.ParseError( - _( - "'%(export_type)s' format not known or not implemented!" - % {"export_type": export_type} - ) + _(f"'{export_type}' format not known or not implemented!") ) return export_type @@ -298,13 +294,13 @@ def _generate_new_export(request, xform, query, export_type, dataview_pk=False): audit, request, ) - except NoRecordsFoundError: - raise Http404(_("No records found to export")) + except NoRecordsFoundError as e: + raise Http404(_("No records found to export")) from e except J2XException as e: # j2x exception return async_status(FAILED, str(e)) except SPSSIOError as e: - raise exceptions.ParseError(str(e)) + raise exceptions.ParseError(str(e)) from e else: return export @@ -319,8 +315,7 @@ def log_export(request, xform, export_type): log.Actions.EXPORT_DOWNLOADED, request.user, xform.user, - _("Downloaded %(export_type)s export on '%(id_string)s'.") - % {"id_string": xform.id_string, "export_type": export_type.upper()}, + _("Downloaded {export_type.upper()} export on '{id_string}'."), audit, request, ) @@ -333,8 +328,7 @@ def external_export_response(export): """ if isinstance(export, Export) and export.internal_status == Export.SUCCESSFUL: return HttpResponseRedirect(export.export_url) - else: - http_status = status.HTTP_400_BAD_REQUEST + http_status = status.HTTP_400_BAD_REQUEST return Response(json.dumps(export), http_status, content_type="application/json") @@ -345,9 +339,9 @@ def _generate_filename(request, xform, remove_group_name=False, dataview_pk=Fals else: # append group name removed flag otherwise use the form id_string if remove_group_name: - filename = "{}-{}".format(xform.id_string, GROUPNAME_REMOVED_FLAG) + filename = f"{xform.id_string}-{GROUPNAME_REMOVED_FLAG}" elif dataview_pk: - filename = "{}-{}".format(xform.id_string, DATAVIEW_EXPORT) + filename = f"{xform.id_string}-{DATAVIEW_EXPORT}" else: filename = xform.id_string @@ -363,17 +357,17 @@ def _set_start_end_params(request, query): try: if request.GET.get("start"): query[SUBMISSION_TIME]["$gte"] = _format_date_for_mongo( - request.GET["start"], datetime + request.GET["start"] ) if request.GET.get("end"): query[SUBMISSION_TIME]["$lte"] = _format_date_for_mongo( - request.GET["end"], datetime + request.GET["end"] ) - except ValueError: + except ValueError as e: raise exceptions.ParseError( _("Dates must be in the format YY_MM_DD_hh_mm_ss") - ) + ) from e else: query = json.dumps(query) @@ -391,8 +385,11 @@ def _get_extension_from_export_type(export_type): return extension -def _format_date_for_mongo(x, datetime): # pylint: disable=W0621, C0103 - return datetime.strptime(x, "%y_%m_%d_%H_%M_%S").strftime("%Y-%m-%dT%H:%M:%S") +# pylint: disable=invalid-name +def _format_date_for_mongo(datetime_str): + return datetime.strptime(datetime_str, "%y_%m_%d_%H_%M_%S").strftime( + "%Y-%m-%dT%H:%M:%S" + ) def process_async_export(request, xform, export_type, options=None): @@ -497,8 +494,8 @@ def _create_export_async( export, async_result = viewer_task.create_async_export( xform, export_type, query, force_xlsx, options=options ) - except ExportConnectionError: - raise ServiceUnavailable + except ExportConnectionError as e: + raise ServiceUnavailable from e return async_result.task_id @@ -556,7 +553,7 @@ def _get_response(): except (OperationalError, ConnectionError) as e: report_exception("Connection Error", e, sys.exc_info()) if count > 0: - raise ServiceUnavailable + raise ServiceUnavailable from e return get_async_response(job_uuid, request, xform, count + 1) except BacklogLimitExceeded: @@ -566,13 +563,14 @@ def _get_response(): return resp -def response_for_format(data, format=None): # pylint: disable=W0622 +# pylint: disable=redefined-builtin +def response_for_format(data, format=None): """ Return appropriately formatted data in Response(). """ if format == "xml": formatted_data = data.xml - elif format == "xls" or format == "xlsx": + elif format in ("xls", "xlsx"): if not data.xls or not data.xls.storage.exists(data.xls.name): raise Http404() diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index 2a8d99e24c..cf918952ea 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -1,11 +1,17 @@ -from itertools import chain +# -*- coding=utf-8 -*- +""" +CSV export utility functions. +""" from collections import OrderedDict +from itertools import chain -import unicodecsv as csv from django.conf import settings from django.db.models.query import QuerySet from django.utils.translation import ugettext as _ + from six import iteritems + +import unicodecsv as csv from pyxform.question import Question from pyxform.section import RepeatingSection, Section @@ -14,7 +20,6 @@ from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data from onadata.libs.exceptions import NoRecordsFoundError -from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.common_tags import ( ATTACHMENTS, BAMBOO_DATASET_ID, @@ -26,8 +31,13 @@ ID, MEDIA_ALL_RECEIVED, MEDIA_COUNT, + MULTIPLE_SELECT_TYPE, NA_REP, NOTES, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, + SELECT_BIND_TYPE, STATUS, SUBMISSION_TIME, SUBMITTED_BY, @@ -36,17 +46,13 @@ UUID, VERSION, XFORM_ID_STRING, - REVIEW_STATUS, - REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, - SELECT_BIND_TYPE, - REVIEW_DATE, ) from onadata.libs.utils.export_builder import ( get_choice_label, get_value_or_attachment_uri, track_task_progress, ) +from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl # the bind type of select multiples that we use to compare @@ -69,26 +75,33 @@ NO = 0 +# pylint: disable=invalid-name def remove_dups_from_list_maintain_order(lst): + """Removes duplicates from a list and still maintains the order.""" return list(OrderedDict.fromkeys(lst)) def get_prefix_from_xpath(xpath): + """Returns xpath prefix.""" xpath = str(xpath) parts = xpath.rsplit("/", 1) if len(parts) == 1: return None - elif len(parts) == 2: - return "%s/" % parts[0] - else: - raise ValueError("%s cannot be prefixed, it returns %s" % (xpath, str(parts))) + if len(parts) == 2: + return f"{parts[0]}/" + raise ValueError(f"{xpath} cannot be prefixed, it returns {str(parts)}") -def get_labels_from_columns(columns, dd, group_delimiter, language=None): +def get_labels_from_columns(columns, data_dictionary, group_delimiter, language=None): + """Return ``column`` labels""" labels = [] for col in columns: - elem = dd.get_survey_element(col) - label = dd.get_label(col, elem=elem, language=language) if elem else col + elem = data_dictionary.get_survey_element(col) + label = ( + data_dictionary.get_label(col, elem=elem, language=language) + if elem + else col + ) if elem is not None and elem.type == "": label = group_delimiter.join([elem.parent.name, label]) if label == "": @@ -98,11 +111,13 @@ def get_labels_from_columns(columns, dd, group_delimiter, language=None): return labels -def get_column_names_only(columns, dd, group_delimiter): +# pylint: disable=unused-argument +def get_column_names_only(columns, data_dictionary, group_delimiter): + """Return column names as a list.""" new_columns = [] for col in columns: new_col = None - elem = dd.get_survey_element(col) + elem = data_dictionary.get_survey_element(col) if elem is None: new_col = col elif elem.type != "": @@ -114,13 +129,14 @@ def get_column_names_only(columns, dd, group_delimiter): return new_columns +# pylint: disable=unused-argument,too-many-arguments,too-many-locals def write_to_csv( path, rows, columns, columns_with_hxl=None, remove_group_name=False, - dd=None, + data_dictionary=None, group_delimiter=DEFAULT_GROUP_DELIMITER, include_labels=False, include_labels_only=False, @@ -130,6 +146,7 @@ def write_to_csv( index_tags=DEFAULT_INDEX_TAGS, language=None, ): + """Writes ``rows`` to a file in CSV format.""" na_rep = getattr(settings, "NA_REP", NA_REP) encoding = "utf-8-sig" if win_excel_utf8 else "utf-8" with open(path, "wb") as csvfile: @@ -137,8 +154,10 @@ def write_to_csv( # Check if to truncate the group name prefix if not include_labels_only: - if remove_group_name and dd: - new_cols = get_column_names_only(columns, dd, group_delimiter) + if remove_group_name and data_dictionary: + new_cols = get_column_names_only( + columns, data_dictionary, group_delimiter + ) else: new_cols = columns @@ -153,13 +172,14 @@ def write_to_csv( if include_labels or include_labels_only: labels = get_labels_from_columns( - columns, dd, group_delimiter, language=language + columns, data_dictionary, group_delimiter, language=language ) writer.writerow(labels) if include_hxl and columns_with_hxl: hxl_row = [columns_with_hxl.get(col, "") for col in columns] - hxl_row and writer.writerow(hxl_row) + if hxl_row: + writer.writerow(hxl_row) for i, row in enumerate(rows, start=1): for col in AbstractDataFrameBuilder.IGNORED_COLUMNS: @@ -168,7 +188,12 @@ def write_to_csv( track_task_progress(i, total_records) -class AbstractDataFrameBuilder(object): +# pylint: disable=too-few-public-methods,too-many-instance-attributes +class AbstractDataFrameBuilder: + """ + Abstract Data Frame Builder class + """ + IGNORED_COLUMNS = [ XFORM_ID_STRING, STATUS, @@ -199,6 +224,7 @@ class AbstractDataFrameBuilder(object): Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML """ + # pylint: disable=too-many-arguments,too-many-locals def __init__( self, username, @@ -229,6 +255,7 @@ def __init__( self.filter_query = filter_query self.group_delimiter = group_delimiter self.split_select_multiples = split_select_multiples + # pylint: disable=invalid-name self.BINARY_SELECT_MULTIPLES = binary_select_multiples self.VALUE_SELECT_MULTIPLES = value_select_multiples self.start = start @@ -262,9 +289,9 @@ def __init__( ): raise ValueError( _( - "Invalid option for repeat_index_tags: %s " + f"Invalid option for repeat_index_tags: {index_tags} " "expecting a tuple with opening and closing tags " - "e.g repeat_index_tags=('[', ']')" % index_tags + "e.g repeat_index_tags=('[', ']')" ) ) self.index_tags = index_tags @@ -274,24 +301,26 @@ def __init__( self._setup() def _setup(self): - self.dd = self.xform - self.select_multiples = self._collect_select_multiples(self.dd, self.language) - self.gps_fields = self._collect_gps_fields(self.dd) + self.data_dictionary = self.xform + self.select_multiples = self._collect_select_multiples( + self.data_dictionary, self.language + ) + self.gps_fields = self._collect_gps_fields(self.data_dictionary) @classmethod - def _fields_to_select(cls, dd): + def _fields_to_select(cls, data_dictionary): return [ c.get_abbreviated_xpath() - for c in dd.get_survey_elements() + for c in data_dictionary.get_survey_elements() if isinstance(c, Question) ] @classmethod - def _collect_select_multiples(cls, dd, language=None): + def _collect_select_multiples(cls, data_dictionary, language=None): select_multiples = [] select_multiple_elements = [ e - for e in dd.get_survey_elements_with_choices() + for e in data_dictionary.get_survey_elements_with_choices() if e.bind.get("type") == SELECT_BIND_TYPE and e.type == MULTIPLE_SELECT_TYPE ] for e in select_multiple_elements: @@ -300,7 +329,7 @@ def _collect_select_multiples(cls, dd, language=None): ( c.get_abbreviated_xpath(), c.name, - get_choice_label(c.label, dd, language), + get_choice_label(c.label, data_dictionary, language), ) for c in e.children ] @@ -310,13 +339,15 @@ def _collect_select_multiples(cls, dd, language=None): if ( (not choices and e.choice_filter) or is_choice_randomized ) and e.itemset: - itemset = dd.survey.to_json_dict()["choices"].get(e.itemset) + itemset = data_dictionary.survey.to_json_dict()["choices"].get( + e.itemset + ) choices = ( [ ( "/".join([xpath, i.get("name")]), i.get("name"), - get_choice_label(i.get("label"), dd, language), + get_choice_label(i.get("label"), data_dictionary, language), ) for i in itemset ] @@ -327,6 +358,7 @@ def _collect_select_multiples(cls, dd, language=None): return dict(select_multiples) + # pylint: disable=too-many-arguments @classmethod def _split_select_multiples( cls, @@ -346,58 +378,43 @@ def _split_select_multiples( if key in record: # split selected choices by spaces and join by / to the # element's xpath - selections = ["%s/%s" % (key, r) for r in record[key].split(" ")] + selections = [f"{key}/{r}" for r in record[key].split(" ")] if value_select_multiples: record.update( - dict( - [ - ( - choice.replace("/" + name, "/" + label) - if show_choice_labels - else choice, - ( - label - if show_choice_labels - else record[key].split()[ - selections.index(choice) - ] - ) - if choice in selections - else None, - ) - for choice, name, label in choices - ] - ) + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: ( + label + if show_choice_labels + else record[key].split()[selections.index(choice)] + ) + if choice in selections + else None + for choice, name, label in choices + } ) elif not binary_select_multiples: # add columns to record for every choice, with default # False and set to True for items in selections record.update( - dict( - [ - ( - choice.replace("/" + name, "/" + label) - if show_choice_labels - else choice, - choice in selections, - ) - for choice, name, label in choices - ] - ) + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: choice in selections + for choice, name, label in choices + } ) else: record.update( - dict( - [ - ( - choice.replace("/" + name, "/" + label) - if show_choice_labels - else choice, - YES if choice in selections else NO, - ) - for choice, name, label in choices - ] - ) + { + choice.replace("/" + name, "/" + label) + if show_choice_labels + else choice: YES + if choice in selections + else NO + for choice, name, label in choices + } ) # remove the column since we are adding separate columns # for each choice @@ -418,10 +435,10 @@ def _split_select_multiples( return record @classmethod - def _collect_gps_fields(cls, dd): + def _collect_gps_fields(cls, data_dictionary): return [ e.get_abbreviated_xpath() - for e in dd.get_survey_elements() + for e in data_dictionary.get_survey_elements() if e.bind.get("type") == "geopoint" ] @@ -434,7 +451,7 @@ def _tag_edit_string(cls, record): tags = [] for tag in record["_tags"]: if "," in tag and " " in tag: - tags.append('"%s"' % tag) + tags.append(f'"{tag}"') else: tags.append(tag) record.update({"_tags": ", ".join(sorted(tags))}) @@ -445,7 +462,7 @@ def _split_gps_fields(cls, record, gps_fields): for (key, value) in iteritems(record): if key in gps_fields and isinstance(value, str): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key) - gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) + gps_parts = {xpath: None for xpath in gps_xpaths} # hack, check if its a list and grab the object within that parts = value.split(" ") # TODO: check whether or not we can have a gps recording @@ -461,6 +478,7 @@ def _split_gps_fields(cls, record, gps_fields): cls._split_gps_fields(list_item, gps_fields) record.update(updated_gps_fields) + # pylint: disable=too-many-arguments def _query_data( self, query="{}", @@ -487,26 +505,30 @@ def _query_data( # if count was requested, return the count if count: return record_count - else: - query_args = { - "xform": self.xform, - "query": query, - "fields": fields, - "start": self.start, - "end": self.end, - # TODO: we might want to add this in for the user - # to sepcify a sort order - "sort": "id", - "start_index": start, - "limit": limit, - "count": False, - } - cursor = query_data(**query_args) - - return cursor + + query_args = { + "xform": self.xform, + "query": query, + "fields": fields, + "start": self.start, + "end": self.end, + # TODO: we might want to add this in for the user + # to sepcify a sort order + "sort": "id", + "start_index": start, + "limit": limit, + "count": False, + } + cursor = query_data(**query_args) + + return cursor +# pylint: disable=too-few-public-methods class CSVDataFrameBuilder(AbstractDataFrameBuilder): + """CSV data frame builder""" + + # pylint: disable=too-many-arguments,too-many-locals def __init__( self, username, @@ -532,7 +554,7 @@ def __init__( language=None, ): - super(CSVDataFrameBuilder, self).__init__( + super().__init__( username, id_string, filter_query, @@ -558,12 +580,12 @@ def __init__( self.ordered_columns = OrderedDict() self.image_xpaths = ( - [] if not self.include_images else self.dd.get_media_survey_xpaths() + [] + if not self.include_images + else self.data_dictionary.get_media_survey_xpaths() ) - def _setup(self): - super(CSVDataFrameBuilder, self)._setup() - + # pylint: disable=too-many-arguments,too-many-branches,too-many-locals @classmethod def _reindex( cls, @@ -593,14 +615,17 @@ def get_ordered_repeat_value(xpath, repeat_value): for elem in children: if not question_types_to_exclude(elem.type): - xp = elem.get_abbreviated_xpath() - item[xp] = repeat_value.get(xp, DEFAULT_NA_REP) + abbreviated_xpath = elem.get_abbreviated_xpath() + item[abbreviated_xpath] = repeat_value.get( + abbreviated_xpath, DEFAULT_NA_REP + ) return item - d = {} + record = {} # check for lists + # pylint: disable=too-many-nested-blocks if ( isinstance(value, list) and len(value) > 0 @@ -625,28 +650,18 @@ def get_ordered_repeat_value(xpath, repeat_value): parent_prefix + key.split("/")[len(parent_prefix) :] ) xpaths = [ - "{key}{open_tag}{index}{close_tag}".format( - key=_key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1], - ) + f"{_key}{index_tags[0]}{index}{index_tags[1]}" ] + nested_key.split("/")[len(_key.split("/")) :] else: xpaths = [ - "{key}{open_tag}{index}{close_tag}".format( - key=key, - open_tag=index_tags[0], - index=index, - close_tag=index_tags[1], - ) + f"{key}{index_tags[0]}{index}{index_tags[1]}" ] + nested_key.split("/")[len(key.split("/")) :] # re-create xpath the split on / xpaths = "/".join(xpaths).split("/") new_prefix = xpaths[:-1] if isinstance(nested_val, list): # if nested_value is a list, rinse and repeat - d.update( + record.update( cls._reindex( nested_key, nested_val, @@ -669,7 +684,7 @@ def get_ordered_repeat_value(xpath, repeat_value): if key in list(ordered_columns): if new_xpath not in ordered_columns[key]: ordered_columns[key].append(new_xpath) - d[new_xpath] = get_value_or_attachment_uri( + record[new_xpath] = get_value_or_attachment_uri( nested_key, nested_val, row, @@ -679,7 +694,7 @@ def get_ordered_repeat_value(xpath, repeat_value): language=language, ) else: - d[key] = get_value_or_attachment_uri( + record[key] = get_value_or_attachment_uri( key, value, row, @@ -693,9 +708,9 @@ def get_ordered_repeat_value(xpath, repeat_value): # safe to simply assign if key == NOTES: # Do not include notes - d[key] = "" + record[key] = "" else: - d[key] = get_value_or_attachment_uri( + record[key] = get_value_or_attachment_uri( key, value, row, @@ -704,7 +719,7 @@ def get_ordered_repeat_value(xpath, repeat_value): show_choice_labels=show_choice_labels, language=language, ) - return d + return record @classmethod def _build_ordered_columns( @@ -717,7 +732,6 @@ def _build_ordered_columns( are not considered columns """ for child in survey_element.children: - # child_xpath = child.get_abbreviated_xpath() if isinstance(child, Section): child_is_repeating = False if isinstance(child, RepeatingSection): @@ -757,7 +771,7 @@ def _update_ordered_columns_from_data(self, cursor): # add ordered columns for gps fields for key in self.gps_fields: - gps_xpaths = self.dd.get_additional_geopoint_xpaths(key) + gps_xpaths = self.data_dictionary.get_additional_geopoint_xpaths(key) self.ordered_columns[key] = [key] + gps_xpaths # add ordered columns for nested repeat data @@ -769,7 +783,7 @@ def _update_ordered_columns_from_data(self, cursor): value, self.ordered_columns, record, - self.dd, + self.data_dictionary, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, @@ -804,7 +818,7 @@ def _format_for_dataframe(self, cursor): value, self.ordered_columns, record, - self.dd, + self.data_dictionary, include_images=self.image_xpaths, split_select_multiples=self.split_select_multiples, index_tags=self.index_tags, @@ -816,7 +830,7 @@ def _format_for_dataframe(self, cursor): def export_to(self, path, dataview=None): self.ordered_columns = OrderedDict() - self._build_ordered_columns(self.dd.survey, self.ordered_columns) + self._build_ordered_columns(self.data_dictionary.survey, self.ordered_columns) if dataview: cursor = dataview.query_data( @@ -863,15 +877,15 @@ def export_to(self, path, dataview=None): ) # add extra columns - columns += [col for col in self.extra_columns] + columns += list(self.extra_columns) - for field in self.dd.get_survey_elements_of_type("osm"): + for field in self.data_dictionary.get_survey_elements_of_type("osm"): columns += OsmData.get_tag_keys( self.xform, field.get_abbreviated_xpath(), include_prefix=True ) columns_with_hxl = self.include_hxl and get_columns_with_hxl( - self.dd.survey_elements + self.data_dictionary.survey_elements ) write_to_csv( @@ -880,7 +894,7 @@ def export_to(self, path, dataview=None): columns, columns_with_hxl=columns_with_hxl, remove_group_name=self.remove_group_name, - dd=self.dd, + data_dictionary=self.data_dictionary, group_delimiter=self.group_delimiter, include_labels=self.include_labels, include_labels_only=self.include_labels_only, diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index f4a1c17839..45601a9877 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -23,17 +23,18 @@ def _get_xml_obj(xml): if not isinstance(xml, bytes): xml = xml.strip().encode() try: - return etree.fromstring(xml) # pylint: disable=E1101 - except etree.XMLSyntaxError as e: # pylint: disable=E1101 + return etree.fromstring(xml) # pylint: disable=no-member + except etree.XMLSyntaxError as e: # pylint: disable=no-member if "Attribute action redefined" in e.msg: xml = xml.replace(b'action="modify" ', b"") return _get_xml_obj(xml) + return None def _get_node(ref, root): point = None - nodes = root.xpath('//node[@id="{}"]'.format(ref)) + nodes = root.xpath(f'//node[@id="{ref}"]') if nodes: node = nodes[0] point = Point(float(node.get("lon")), float(node.get("lat"))) @@ -61,11 +62,11 @@ def get_combined_osm(osm_list): for child in _osm.getchildren(): osm.append(child) if osm is not None: - # pylint: disable=E1101 + # pylint: disable=no-member return etree.tostring(osm, encoding="utf-8", xml_declaration=True) - elif isinstance(osm_list, dict): + if isinstance(osm_list, dict): if "detail" in osm_list: - xml = "%s" % osm_list["detail"] + xml = f"{osm_list['detail']}" return xml.encode("utf-8") @@ -166,7 +167,7 @@ def save_osm_data(instance_id): if isinstance(osm_xml, bytes): osm_xml = osm_xml.decode("utf-8") except IOError as e: - logging.exception("IOError saving osm data: %s" % str(e)) + logging.exception("IOError saving osm data: %s", str(e)) continue else: filename = None @@ -226,6 +227,6 @@ def osm_flat_dict(instance_id): for osm in osm_data: for tag in osm.tags: for (k, v) in iteritems(tag): - tags.update({"osm_{}".format(k): v}) + tags.update({f"osm_{k}": v}) return tags From a4268a05ae77f4072231c150a9f98b933215afe0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 22:48:10 +0300 Subject: [PATCH 099/234] export_builder.py: cleanup --- onadata/libs/utils/export_builder.py | 455 +++++++++++++++------------ 1 file changed, 251 insertions(+), 204 deletions(-) diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index c54483a5e0..7d916b1cf5 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines # -*- coding: utf-8 -*- """ ExportBuilder @@ -5,25 +6,24 @@ from __future__ import unicode_literals import csv -import logging -import sys -import uuid import re -from builtins import str as text -from datetime import datetime, date -from zipfile import ZipFile, ZIP_DEFLATED +import uuid +from datetime import date, datetime +from zipfile import ZIP_DEFLATED, ZipFile -from celery import current_task from django.conf import settings +from django.contrib.sites.models import Site from django.core.files.temp import NamedTemporaryFile -from onadata.libs.utils.common_tools import str_to_bool -from django.utils.translation import ugettext as _ + from six import iteritems + +from celery import current_task from openpyxl.utils.datetime import to_excel from openpyxl.workbook import Workbook from pyxform.question import Question from pyxform.section import RepeatingSection, Section + try: from savReaderWriter import SavWriter except ImportError: @@ -44,13 +44,17 @@ ID, INDEX, MULTIPLE_SELECT_TYPE, - SELECT_ONE, NOTES, PARENT_INDEX, PARENT_TABLE_NAME, REPEAT_INDEX_TAGS, + REVIEW_COMMENT, + REVIEW_DATE, + REVIEW_STATUS, SAV_255_BYTES_TYPE, SAV_NUMERIC_TYPE, + SELECT_BIND_TYPE, + SELECT_ONE, STATUS, SUBMISSION_TIME, SUBMITTED_BY, @@ -58,11 +62,8 @@ UUID, VERSION, XFORM_ID_STRING, - REVIEW_STATUS, - REVIEW_COMMENT, - SELECT_BIND_TYPE, - REVIEW_DATE, ) +from onadata.libs.utils.common_tools import str_to_bool from onadata.libs.utils.mongo import _decode_from_mongo, _is_invalid_for_mongo # the bind type of select multiples that we use to compare @@ -73,9 +74,6 @@ YES = 1 NO = 0 -# savReaderWriter behaves differenlty depending on this -IS_PY_3K = sys.version_info[0] > 2 - def current_site_url(path): """ @@ -83,16 +81,15 @@ def current_site_url(path): :param path :return: complete url """ - from django.contrib.sites.models import Site current_site = Site.objects.get_current() protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") port = getattr(settings, "ONA_SITE_PORT", "") - url = "%s://%s" % (protocol, current_site.domain) + url = f"{protocol}://{current_site.domain}" if port: - url += ":%s" % port + url += f":{port}" if path: - url += "%s" % path + url += f"{path}" return url @@ -102,7 +99,7 @@ def get_choice_label(label, data_dictionary, language=None): Return the label matching selected language or simply just the label. """ if isinstance(label, dict): - languages = [i for i in label.keys()] + languages = list(label.keys()) _language = ( language if language in languages @@ -144,7 +141,8 @@ def _get_choice_label_value(lookup): return label or value -def get_value_or_attachment_uri( # pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments +def get_value_or_attachment_uri( key, value, row, @@ -183,13 +181,16 @@ def get_value_or_attachment_uri( # pylint: disable=too-many-arguments def get_data_dictionary_from_survey(survey): - dd = DataDictionary() - dd._survey = survey + """Creates a DataDictionary instance from an XML survey instance.""" + data_dicionary = DataDictionary() + # pylint: disable=protected-access + data_dicionary._survey = survey - return dd + return data_dicionary def encode_if_str(row, key, encode_dates=False, sav_writer=None): + """Encode a string value in ``row[key]``.""" val = row.get(key) if isinstance(val, (datetime, date)): if sav_writer: @@ -203,16 +204,17 @@ def encode_if_str(row, key, encode_dates=False, sav_writer=None): return sav_writer.spssDateTime( val.isoformat().encode("utf-8"), strptime_fmt ) - elif encode_dates: + if encode_dates: return val.isoformat() if sav_writer: val = "" if val is None else val - return text(val) if IS_PY_3K and not isinstance(val, bool) else val + return str(val) if not isinstance(val, bool) else val return val -def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths=[]): +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths=None): """ Converts a dict into one or more tabular datasets :param data: current record which can be changed or updated @@ -223,7 +225,8 @@ def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths= :param row: current record that remains unchanged on this function's recall """ output = {} - # TODO: test for _geolocation and attachment lists + media_xpaths = [] if media_xpaths is None else media_xpaths + # pylint: disable=too-many-nested-blocks if isinstance(data, dict): for (key, val) in iteritems(data): if isinstance(val, list) and key not in [NOTES, ATTACHMENTS, TAGS]: @@ -236,7 +239,7 @@ def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths= new_output = dict_to_joined_export( child, child_index, indices, key, survey, row, media_xpaths ) - d = { + item = { INDEX: child_index, PARENT_INDEX: index, PARENT_TABLE_NAME: name, @@ -249,15 +252,15 @@ def dict_to_joined_export(data, index, indices, name, survey, row, media_xpaths= output[out_key] = [] output[out_key].extend(out_val) else: - d.update(out_val) - output[key].append(d) + item.update(out_val) + output[key].append(item) else: if name not in output: output[name] = {} if key in [TAGS]: output[name][key] = ",".join(val) elif key in [NOTES]: - note_list = [v if isinstance(v, text) else v["note"] for v in val] + note_list = [v if isinstance(v, str) else v["note"] for v in val] output[name][key] = "\r\n".join(note_list) else: data_dictionary = get_data_dictionary_from_survey(survey) @@ -285,7 +288,7 @@ def is_all_numeric(items): for i in items: float(i) # if there is a zero padded number, it is not all numeric - if isinstance(i, text) and len(i) > 1 and i[0] == "0" and i[1] != ".": + if isinstance(i, str) and len(i) > 1 and i[0] == "0" and i[1] != ".": return False return True except ValueError: @@ -294,11 +297,9 @@ def is_all_numeric(items): # check for zero padded numbers to be treated as non numeric return not ( any( - [ - i.startswith("0") and len(i) > 1 and i.find(".") == -1 - for i in items - if isinstance(i, text) - ] + i.startswith("0") and len(i) > 1 and i.find(".") == -1 + for i in items + if isinstance(i, str) ) ) @@ -312,29 +313,24 @@ def track_task_progress(additions, total=None): :param total: :return: """ - try: - if ( - additions - % getattr( - settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH - ) - == 0 - ): - meta = {"progress": additions} - if total: - meta.update({"total": total}) - current_task.update_state(state="PROGRESS", meta=meta) - except Exception as e: - logging.exception(_("Track task progress threw exception: %s" % text(e))) + batch_size = getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH + ) + if additions % batch_size == 0: + meta = {"progress": additions} + if total: + meta.update({"total": total}) + current_task.update_state(state="PROGRESS", meta=meta) +# pylint: disable=invalid-name def string_to_date_with_xls_validation(date_str): """Try to convert a string to a date object. :param date_str: string to convert :returns: object if converted, otherwise date string """ - if not isinstance(date_str, text): + if not isinstance(date_str, str): return date_str try: @@ -346,6 +342,7 @@ def string_to_date_with_xls_validation(date_str): return date_obj +# pylint: disable=invalid-name def decode_mongo_encoded_section_names(data): """Recursively decode mongo keys. @@ -365,7 +362,50 @@ def decode_mongo_encoded_section_names(data): return results -class ExportBuilder(object): +def _check_sav_column(column, columns): + """ + Check for duplicates and append @ 4 chars uuid. + Also checks for column length more than 64 chars + :param column: + :return: truncated column + """ + + if len(column) > 64: + col_len_diff = len(column) - 64 + column = column[:-col_len_diff] + + if column.lower() in (t.lower() for t in columns): + if len(column) > 59: + column = column[:-5] + column = column + "@" + str(uuid.uuid4()).split("-")[1] + + return column + + +def _get_var_name(title, var_names): + """ + GET valid SPSS varName. + @param title - survey element title/name + @param var_names - list of existing var_names + @return valid varName and list of var_names with new var name appended + """ + var_name = ( + title.replace("/", ".") + .replace("-", "_") + .replace(":", "_") + .replace("{", "") + .replace("}", "") + ) + var_name = _check_sav_column(var_name, var_names) + var_name = "@" + var_name if var_name.startswith("_") else var_name + var_names.append(var_name) + return var_name, var_names + + +# pylint: disable=too-many-instance-attributes +class ExportBuilder: + """Utility class for generating multiple formats of data export to file.""" + IGNORED_COLUMNS = [ XFORM_ID_STRING, STATUS, @@ -435,6 +475,7 @@ def format_field_title( data_dictionary, remove_group_name=False, ): + """Format the field title.""" title = abbreviated_xpath # Check if to truncate the group name prefix if remove_group_name: @@ -453,6 +494,7 @@ def format_field_title( return title def get_choice_label_from_dict(self, label): + """Returns the choice label for the default language.""" if isinstance(label, dict): language = self.get_default_language(list(label)) label = label.get(self.language or language) @@ -460,11 +502,11 @@ def get_choice_label_from_dict(self, label): return label def _get_select_mulitples_choices( - self, child, dd, field_delimiter, remove_group_name + self, child, data_dicionary, field_delimiter, remove_group_name ): def get_choice_dict(xpath, label): title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, remove_group_name + xpath, field_delimiter, data_dicionary, remove_group_name ) return { @@ -483,7 +525,7 @@ def get_choice_dict(xpath, label): if ( (not child.children and child.choice_filter) or is_choice_randomized ) and child.itemset: - itemset = dd.survey.to_json_dict()["choices"].get(child.itemset) + itemset = data_dicionary.survey.to_json_dict()["choices"].get(child.itemset) choices = ( [ get_choice_dict( @@ -499,7 +541,7 @@ def get_choice_dict(xpath, label): choices = [ get_choice_dict( c.get_abbreviated_xpath(), - get_choice_label(c.label, dd, language=self.language), + get_choice_label(c.label, data_dicionary, language=self.language), ) for c in child.children ] @@ -507,15 +549,18 @@ def get_choice_dict(xpath, label): return choices def set_survey(self, survey, xform=None, include_reviews=False): + """Set's the XForm XML ``survey`` instance.""" if self.INCLUDE_REVIEWS or include_reviews: + # pylint: disable=invalid-name self.EXTRA_FIELDS = self.EXTRA_FIELDS + [ REVIEW_STATUS, REVIEW_COMMENT, REVIEW_DATE, ] self.__init__() - dd = get_data_dictionary_from_survey(survey) + data_dicionary = get_data_dictionary_from_survey(survey) + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments def build_sections( current_section, survey_element, @@ -529,6 +574,7 @@ def build_sections( remove_group_name=False, language=None, ): + # pylint: disable=too-many-nested-blocks for child in survey_element.children: current_section_name = current_section["name"] # if a section, recurs @@ -579,11 +625,13 @@ def build_sections( _title = ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter, - dd, + data_dicionary, remove_group_name, ) _label = ( - dd.get_label(child_xpath, elem=child, language=language) + data_dicionary.get_label( + child_xpath, elem=child, language=language + ) or _title ) current_section["elements"].append( @@ -610,7 +658,10 @@ def build_sections( choices = [] if self.SPLIT_SELECT_MULTIPLES: choices = self._get_select_mulitples_choices( - child, dd, field_delimiter, remove_group_name + child, + data_dicionary, + field_delimiter, + remove_group_name, ) for choice in choices: if choice not in current_section["elements"]: @@ -632,7 +683,10 @@ def build_sections( ) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, remove_group_name + xpath, + field_delimiter, + data_dicionary, + remove_group_name, ) current_section["elements"].append( { @@ -654,7 +708,10 @@ def build_sections( xpaths = _get_osm_paths(child, xform) for xpath in xpaths: _title = ExportBuilder.format_field_title( - xpath, field_delimiter, dd, remove_group_name + xpath, + field_delimiter, + data_dicionary, + remove_group_name, ) current_section["elements"].append( { @@ -698,7 +755,8 @@ def _get_osm_paths(osm_field, xform): ) return osm_columns - self.dd = dd + # pylint: disable=attribute-defined-outside-init + self.data_dicionary = data_dicionary self.survey = survey self.select_multiples = {} self.select_ones = {} @@ -722,6 +780,7 @@ def _get_osm_paths(osm_field, xform): ) def section_by_name(self, name): + """Return section by the given ``name``.""" matches = [s for s in self.sections if s["name"] == name] assert len(matches) == 1 @@ -761,12 +820,10 @@ def split_select_multiples( # for each select_multiple, get the associated data and split it for (xpath, choices) in iteritems(select_multiples): # get the data matching this xpath - data = row.get(xpath) and text(row.get(xpath)) + data = row.get(xpath) and str(row.get(xpath)) selections = [] if data: - selections = [ - "{0}/{1}".format(xpath, selection) for selection in data.split() - ] + selections = [f"{xpath}/{selection}" for selection in data.split()] if show_choice_labels and data_dictionary: row[xpath] = get_choice_label_value( xpath, data, data_dictionary, language @@ -774,59 +831,45 @@ def split_select_multiples( if select_values: if show_choice_labels: row.update( - dict( - [ - ( - choice["label"], - choice["_label"] - if selections and choice["xpath"] in selections - else None, - ) - for choice in choices - ] - ) + { + choice["label"]: choice["_label"] + if selections and choice["xpath"] in selections + else None + for choice in choices + } ) else: row.update( - dict( - [ - ( - choice["xpath"], - data.split()[selections.index(choice["xpath"])] - if selections and choice["xpath"] in selections - else None, - ) - for choice in choices + { + choice["xpath"]: data.split()[ + selections.index(choice["xpath"]) ] - ) + if selections and choice["xpath"] in selections + else None + for choice in choices + } ) elif binary_select_multiples: row.update( - dict( - [ - ( - choice["label"] - if show_choice_labels - else choice["xpath"], - YES if choice["xpath"] in selections else NO, - ) - for choice in choices - ] - ) + { + choice["label"] + if show_choice_labels + else choice["xpath"]: YES + if choice["xpath"] in selections + else NO + for choice in choices + } ) else: row.update( - dict( - [ - ( - choice["label"] - if show_choice_labels - else choice["xpath"], - choice["xpath"] in selections if selections else None, - ) - for choice in choices - ] - ) + { + choice["label"] + if show_choice_labels + else choice["xpath"]: choice["xpath"] in selections + if selections + else None + for choice in choices + } ) return row @@ -863,6 +906,7 @@ def convert_type(cls, value, data_type): except ValueError: return value + # pylint: disable=too-many-branches def pre_process_row(self, row, section): """ Split select multiples, gps and decode . and $ @@ -883,16 +927,16 @@ def pre_process_row(self, row, section): self.VALUE_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES, show_choice_labels=self.SHOW_CHOICE_LABELS, - data_dictionary=self.dd, + data_dictionary=self.data_dicionary, language=self.language, ) if not self.SPLIT_SELECT_MULTIPLES and self.SHOW_CHOICE_LABELS: for xpath in select_multiples: # get the data matching this xpath - data = row.get(xpath) and text(row.get(xpath)) + data = row.get(xpath) and str(row.get(xpath)) if data: row[xpath] = get_choice_label_value( - xpath, data, self.dd, self.language + xpath, data, self.data_dicionary, self.language ) if section_name in self.gps_fields: @@ -902,7 +946,7 @@ def pre_process_row(self, row, section): for key in self.select_ones[section_name]: if key in row: row[key] = get_choice_label_value( - key, row[key], self.dd, self.language + key, row[key], self.data_dicionary, self.language ) # convert to native types @@ -925,7 +969,7 @@ def pre_process_row(self, row, section): # Map dynamic values for key, value in row.items(): if isinstance(value, str): - dynamic_val_regex = "\$\{\w+\}" # noqa + dynamic_val_regex = r"\$\{\w+\}" # noqa # Find substrings that match ${`any_text`} result = re.findall(dynamic_val_regex, value) if result: @@ -939,7 +983,10 @@ def pre_process_row(self, row, section): return row + # pylint: disable=too-many-locals,too-many-branches def to_zipped_csv(self, path, data, *args, **kwargs): + """Export CSV formatted files from ``data`` and zip the files.""" + def write_row(row, csv_writer, fields): csv_writer.writerow([encode_if_str(row, field) for field in fields]) @@ -956,16 +1003,18 @@ def write_row(row, csv_writer, fields): if not self.INCLUDE_LABELS_ONLY: for section in self.sections: fields = self.get_fields(dataview, section, "title") - csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) + csv_defs[section["name"]]["csv_writer"].writerow(list(fields)) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: for section in self.sections: fields = self.get_fields(dataview, section, "label") - csv_defs[section["name"]]["csv_writer"].writerow([f for f in fields]) + csv_defs[section["name"]]["csv_writer"].writerow(list(fields)) media_xpaths = ( - [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() ) columns_with_hxl = kwargs.get("columns_with_hxl") @@ -981,10 +1030,16 @@ def write_row(row, csv_writer, fields): index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): + for i, row_data in enumerate(data, start=1): # decode mongo section names joined_export = dict_to_joined_export( - d, index, indices, survey_name, self.survey, d, media_xpaths + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) @@ -1027,6 +1082,7 @@ def write_row(row, csv_writer, fields): @classmethod def get_valid_sheet_name(cls, desired_name, existing_names): + """Returns a valid sheet_name based on the desired names""" # a sheet name has to be <= 31 characters and not a duplicate of an # existing sheet # truncate sheet_name to XLSDataFrameBuilder.SHEET_NAME_MAX_CHARS @@ -1036,16 +1092,19 @@ def get_valid_sheet_name(cls, desired_name, existing_names): i = 1 generated_name = new_sheet_name while generated_name in existing_names: - digit_length = len(text(i)) + digit_length = len(str(i)) allowed_name_len = cls.XLS_SHEET_NAME_MAX_CHARS - digit_length # make name the required len if len(generated_name) > allowed_name_len: generated_name = generated_name[:allowed_name_len] - generated_name = "{0}{1}".format(generated_name, i) + generated_name = f"{generated_name}{i}" i += 1 return generated_name + # pylint: disable=too-many-locals def to_xls_export(self, path, data, *args, **kwargs): + """Export data to a spreadsheet document.""" + def write_row(data, work_sheet, fields, work_sheet_titles): # update parent_table with the generated sheet's title data[PARENT_TABLE_NAME] = work_sheet_titles.get(data.get(PARENT_TABLE_NAME)) @@ -1054,7 +1113,7 @@ def write_row(data, work_sheet, fields, work_sheet_titles): dataview = kwargs.get("dataview") total_records = kwargs.get("total_records") - wb = Workbook(write_only=True) + work_book = Workbook(write_only=True) work_sheets = {} # map of section_names to generated_names work_sheet_titles = {} @@ -1064,7 +1123,7 @@ def write_row(data, work_sheet, fields, work_sheet_titles): "_".join(section_name.split("/")), work_sheet_titles.values() ) work_sheet_titles[section_name] = work_sheet_title - work_sheets[section_name] = wb.create_sheet(title=work_sheet_title) + work_sheets[section_name] = work_book.create_sheet(title=work_sheet_title) # write the headers if not self.INCLUDE_LABELS_ONLY: @@ -1073,8 +1132,8 @@ def write_row(data, work_sheet, fields, work_sheet_titles): headers = self.get_fields(dataview, section, "title") # get the worksheet - ws = work_sheets[section_name] - ws.append(headers) + work_sheet = work_sheets[section_name] + work_sheet.append(headers) # write labels if self.INCLUDE_LABELS or self.INCLUDE_LABELS_ONLY: @@ -1083,11 +1142,13 @@ def write_row(data, work_sheet, fields, work_sheet_titles): labels = self.get_fields(dataview, section, "label") # get the worksheet - ws = work_sheets[section_name] - ws.append(labels) + work_sheet = work_sheets[section_name] + work_sheet.append(labels) media_xpaths = ( - [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() ) # write hxl header @@ -1098,17 +1159,24 @@ def write_row(data, work_sheet, fields, work_sheet_titles): headers = self.get_fields(dataview, section, "title") # get the worksheet - ws = work_sheets[section_name] + work_sheet = work_sheets[section_name] hxl_row = [columns_with_hxl.get(col, "") for col in headers] - hxl_row and ws.append(hxl_row) + if hxl_row: + work_sheet.append(hxl_row) index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): + for i, row_data in enumerate(data, start=1): joined_export = dict_to_joined_export( - d, index, indices, survey_name, self.survey, d, media_xpaths + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) @@ -1122,14 +1190,14 @@ def write_row(data, work_sheet, fields, work_sheet_titles): section_name = section["name"] fields = self.get_fields(dataview, section, "xpath") - ws = work_sheets[section_name] + work_sheet = work_sheets[section_name] # section might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if isinstance(row, dict): write_row( self.pre_process_row(row, section), - ws, + work_sheet, fields, work_sheet_titles, ) @@ -1137,22 +1205,23 @@ def write_row(data, work_sheet, fields, work_sheet_titles): for child_row in row: write_row( self.pre_process_row(child_row, section), - ws, + work_sheet, fields, work_sheet_titles, ) index += 1 track_task_progress(i, total_records) - wb.save(filename=path) + work_book.save(filename=path) + # pylint: disable=too-many-locals,unused-argument def to_flat_csv_export( self, path, data, username, id_string, filter_query, **kwargs ): """ Generates a flattened CSV file for submitted data. """ - # TODO resolve circular import + # pylint: disable=import-outside-toplevel from onadata.libs.utils.csv_builder import CSVDataFrameBuilder start = kwargs.get("start") @@ -1193,7 +1262,8 @@ def to_flat_csv_export( csv_builder.export_to(path, dataview=dataview) def get_default_language(self, languages): - language = self.dd.default_language + """Return the default languange of the XForm.""" + language = self.data_dicionary.default_language if languages and ((language and language not in languages) or not language): languages.sort() language = languages[0] @@ -1211,22 +1281,25 @@ def _get_sav_value_labels(self, xpath_var_names=None): 'available': {0: 'No', 1: 'Yes'} } """ - choice_questions = self.dd.get_survey_elements_with_choices() + choice_questions = self.data_dicionary.get_survey_elements_with_choices() sav_value_labels = {} - for q in choice_questions: - if xpath_var_names and q.get_abbreviated_xpath() not in xpath_var_names: + for question in choice_questions: + if ( + xpath_var_names + and question.get_abbreviated_xpath() not in xpath_var_names + ): continue var_name = ( - xpath_var_names.get(q.get_abbreviated_xpath()) + xpath_var_names.get(question.get_abbreviated_xpath()) if xpath_var_names - else q["name"] + else question["name"] ) - choices = q.to_json_dict().get("children") + choices = question.to_json_dict().get("children") if choices is None: choices = self.survey.get("choices") - if choices is not None and q.get("itemset"): - choices = choices.get(q.get("itemset")) + if choices is not None and question.get("itemset"): + choices = choices.get(question.get("itemset")) _value_labels = {} if choices: is_numeric = is_all_numeric([c["name"] for c in choices]) @@ -1234,7 +1307,7 @@ def _get_sav_value_labels(self, xpath_var_names=None): name = choice["name"].strip() # should skip select multiple and zero padded numbers e.g # 009 or 09, they should be treated as strings - if q.type != "select all that apply" and is_numeric: + if question.type != "select all that apply" and is_numeric: try: name = ( float(name) if (float(name) > int(name)) else int(name) @@ -1243,29 +1316,11 @@ def _get_sav_value_labels(self, xpath_var_names=None): pass label = self.get_choice_label_from_dict(choice.get("label", "")) _value_labels[name] = label.strip() - sav_value_labels[var_name or q["name"]] = _value_labels + sav_value_labels[var_name or question["name"]] = _value_labels return sav_value_labels - def _get_var_name(self, title, var_names): - """ - GET valid SPSS varName. - @param title - survey element title/name - @param var_names - list of existing var_names - @return valid varName and list of var_names with new var name appended - """ - var_name = ( - title.replace("/", ".") - .replace("-", "_") - .replace(":", "_") - .replace("{", "") - .replace("}", "") - ) - var_name = self._check_sav_column(var_name, var_names) - var_name = "@" + var_name if var_name.startswith("_") else var_name - var_names.append(var_name) - return var_name, var_names - + # pylint: disable=too-many-locals def _get_sav_options(self, elements): """ GET/SET SPSS options. @@ -1285,7 +1340,7 @@ def _is_numeric(xpath, element_type, data_dictionary): var_name = xpath_var_names.get(xpath) or xpath if element_type in ["decimal", "int", "date"]: return True - elif element_type == "string": + if element_type == "string": # check if it is a choice part of multiple choice # type is likely empty string, split multi select is binary element = data_dictionary.get_element(xpath) @@ -1302,8 +1357,7 @@ def _is_numeric(xpath, element_type, data_dictionary): parent_xpath = "/".join(xpath.split("/")[:-1]) parent = data_dictionary.get_element(parent_xpath) return parent and parent.type == MULTIPLE_SELECT_TYPE - else: - return False + return False value_select_multiples = self.VALUE_SELECT_MULTIPLES _var_types = {} @@ -1317,15 +1371,15 @@ def _is_numeric(xpath, element_type, data_dictionary): ] for element in elements: title = element["title"] - _var_name, _var_names = self._get_var_name(title, var_names) + _var_name, _var_names = _get_var_name(title, var_names) var_names = _var_names fields_and_labels.append( (element["title"], element["label"], element["xpath"], _var_name) ) - xpath_var_names = dict( - [(xpath, var_name) for field, label, xpath, var_name in fields_and_labels] - ) + xpath_var_names = { + xpath: var_name for field, label, xpath, var_name in fields_and_labels + } all_value_labels = self._get_sav_value_labels(xpath_var_names) @@ -1354,7 +1408,9 @@ def _get_element_type(element_xpath): ( _var_types[element["xpath"]], SAV_NUMERIC_TYPE - if _is_numeric(element["xpath"], element["type"], self.dd) + if _is_numeric( + element["xpath"], element["type"], self.data_dicionary + ) else SAV_255_BYTES_TYPE, ) for element in elements @@ -1372,7 +1428,7 @@ def _get_element_type(element_xpath): ( x[1], SAV_NUMERIC_TYPE - if _is_numeric(x[0], _get_element_type(x[0]), self.dd) + if _is_numeric(x[0], _get_element_type(x[0]), self.data_dicionary) else SAV_255_BYTES_TYPE, ) for x in duplicate_names @@ -1395,33 +1451,16 @@ def _get_element_type(element_xpath): "ioUtf8": True, } - def _check_sav_column(self, column, columns): - """ - Check for duplicates and append @ 4 chars uuid. - Also checks for column length more than 64 chars - :param column: - :return: truncated column - """ - - if len(column) > 64: - col_len_diff = len(column) - 64 - column = column[:-col_len_diff] - - if column.lower() in (t.lower() for t in columns): - if len(column) > 59: - column = column[:-5] - column = column + "@" + text(uuid.uuid4()).split("-")[1] - - return column - + # pylint: disable=too-many-locals def to_zipped_sav(self, path, data, *args, **kwargs): + """Generates the SPSS zipped file format export.""" if SavWriter is None: # Fail silently return total_records = kwargs.get("total_records") - def write_row(row, csv_writer, fields): + def write_row(row, sav_writer, fields): # replace character for osm fields fields = [field.replace(":", "_") for field in fields] sav_writer.writerow( @@ -1440,16 +1479,24 @@ def write_row(row, csv_writer, fields): sav_defs[section["name"]] = {"sav_file": sav_file, "sav_writer": sav_writer} media_xpaths = ( - [] if not self.INCLUDE_IMAGES else self.dd.get_media_survey_xpaths() + [] + if not self.INCLUDE_IMAGES + else self.data_dicionary.get_media_survey_xpaths() ) index = 1 indices = {} survey_name = self.survey.name - for i, d in enumerate(data, start=1): + for i, row_data in enumerate(data, start=1): # decode mongo section names joined_export = dict_to_joined_export( - d, index, indices, survey_name, self.survey, d, media_xpaths + row_data, + index, + indices, + survey_name, + self.survey, + row_data, + media_xpaths, ) output = decode_mongo_encoded_section_names(joined_export) # attach meta fields (index, parent_index, parent_table) From ebde8e2849519ec86555e6f3529d0be205a5dbd5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 2 May 2022 22:49:15 +0300 Subject: [PATCH 100/234] Fix: mock the builtin ConnectionError for connection_error test --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 2 -- onadata/libs/serializers/xform_serializer.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b7be15519a..f8e0399099 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -3481,8 +3481,6 @@ def test_export_zip_async(self, async_result): @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_export_async_connection_error(self, async_result): with HTTMock(enketo_mock): - from requests import ConnectionError - async_result.side_effect = ConnectionError( "Error opening socket: a socket error occurred" ) diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 9dcb3761ff..b381e72306 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -599,6 +599,8 @@ def get_manifest_url(self, obj): class XFormManifestSerializer(serializers.Serializer): + """XForm Manifest serializer class.""" + filename = serializers.SerializerMethodField() hash = serializers.SerializerMethodField() downloadUrl = serializers.SerializerMethodField("get_url") # noqa From b65410358712f1bebe2d9634a881be3ee0dadcdc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 3 May 2022 00:48:53 +0300 Subject: [PATCH 101/234] fix: 'ExportBuilder' object has no attribute 'dd' dd has been renamed to data_dictionary --- onadata/apps/logger/xform_instance_parser.py | 8 +++++--- onadata/libs/tests/utils/test_export_tools.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index e5f423ba1a..cb4cdcae48 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -338,7 +338,7 @@ class XFormInstanceParser: def __init__(self, xml_str, data_dictionary): # pylint: disable=invalid-name - self.dd = data_dictionary + self.data_dicionary = data_dictionary self.parse(xml_str) def parse(self, xml_str): @@ -349,10 +349,12 @@ def parse(self, xml_str): self._root_node = self._xml_obj.documentElement repeats = [ e.get_abbreviated_xpath() - for e in self.dd.get_survey_elements_of_type("repeat") + for e in self.data_dicionary.get_survey_elements_of_type("repeat") ] - self._dict = _xml_node_to_dict(self._root_node, repeats, self.dd.encrypted) + self._dict = _xml_node_to_dict( + self._root_node, repeats, self.data_dicionary.encrypted + ) self._flat_dict = {} if self._dict is None: diff --git a/onadata/libs/tests/utils/test_export_tools.py b/onadata/libs/tests/utils/test_export_tools.py index 81bc42ca6d..6db165f9b9 100644 --- a/onadata/libs/tests/utils/test_export_tools.py +++ b/onadata/libs/tests/utils/test_export_tools.py @@ -613,7 +613,7 @@ def test_get_sav_value_labels_multi_language(self): expected_data = {"fruit": {"orange": "Orange", "mango": "Mango"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) - export_builder.dd._default_language = "Swahili" + export_builder.data_dicionary._default_language = "Swahili" expected_data = {"fruit": {"orange": "Chungwa", "mango": "Maembe"}} self.assertEqual(export_builder._get_sav_value_labels(), expected_data) From eb277bcc82fe326331c66ed7f42b310c661cea49 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 3 May 2022 18:14:40 +0300 Subject: [PATCH 102/234] batch: cleanup --- onadata/apps/api/viewsets/data_viewset.py | 2 + onadata/apps/logger/models/project.py | 36 ++++++++++---- onadata/apps/main/forms.py | 48 +++++++++++-------- .../serializers/password_reset_serializer.py | 6 ++- onadata/libs/utils/csv_builder.py | 7 +-- 5 files changed, 61 insertions(+), 38 deletions(-) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 2ddf538979..18196a292e 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -687,6 +687,7 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): raise ParseError(str(e)) from e def paginate_queryset(self, queryset): + """Returns a paginated queryset.""" if self.paginator is None: return None return self.paginator.paginate_queryset( @@ -748,6 +749,7 @@ def _get_streaming_response(self): """ def get_json_string(item): + """Returns the ``item`` Instance instance as a JSON string.""" return json.dumps(item.json if isinstance(item, Instance) else item) if self.kwargs.get("format") == "xml": diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index c0ff477d57..3dc7703c64 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -3,10 +3,10 @@ Project model class """ from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.db import models, transaction -from django.db.models import Prefetch, JSONField +from django.db.models import Prefetch from django.db.models.signals import post_save from django.utils import timezone @@ -17,14 +17,24 @@ from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.common_tags import OWNER_TEAM_NAME +# pylint: disable=invalid-name +User = get_user_model() + +# pylint: disable=too-few-public-methods class PrefetchManager(models.Manager): + """Project prefetched manager - prefetches models related to the Project model.""" + def get_queryset(self): - from onadata.apps.logger.models.xform import XForm + """Return a queryset with the XForm, Team, tags, and other related relations + prefetched.""" + # pylint: disable=import-outside-toplevel from onadata.apps.api.models.team import Team + from onadata.apps.logger.models.xform import XForm + # pylint: disable=no-member return ( - super(PrefetchManager, self) + super() .get_queryset() .select_related("created_by", "organization") .prefetch_related( @@ -83,7 +93,8 @@ class Project(BaseModel): """ name = models.CharField(max_length=255) - metadata = JSONField(default=dict) + # pylint: disable=no-member + metadata = models.JSONField(default=dict) organization = models.ForeignKey( settings.AUTH_USER_MODEL, related_name="project_org", on_delete=models.CASCADE ) @@ -123,21 +134,22 @@ class Meta: ) def __str__(self): - return "%s|%s" % (self.organization, self.name) + return f"{self.organization}|{self.name}" def clean(self): - # pylint: disable=E1101 + """Raises a validation error if a project with same name and organization exists.""" query_set = Project.objects.exclude(pk=self.pk).filter( name__iexact=self.name, organization=self.organization ) if query_set.exists(): raise ValidationError( - 'Project name "%s" is already in' - " use in this account." % self.name.lower() + f'Project name "{self.name.lower()}" is already in' + " use in this account." ) @property def user(self): + """Returns the user who created the project.""" return self.created_by @transaction.atomic() @@ -161,13 +173,15 @@ def soft_delete(self, user=None): form.soft_delete(user=user) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """Sets permissions to users who are owners of the organization.""" if created: for perm in get_perms_for_model(Project): assign_perm(perm.codename, instance.organization, instance) owners = instance.organization.team_set.filter( - name="{}#{}".format(instance.organization.username, OWNER_TEAM_NAME), + name=f"{instance.organization.username}#{OWNER_TEAM_NAME}", organization=instance.organization, ) for owner in owners: @@ -187,12 +201,14 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): ) +# pylint: disable=too-few-public-methods class ProjectUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" content_object = models.ForeignKey(Project, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class ProjectGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index 0947297ff5..9797d24731 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -3,14 +3,14 @@ forms module. """ import os +import random import re from six.moves.urllib.parse import urlparse -from six.moves.urllib.request import urlopen import requests from django import forms from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.base import ContentFile from django.core.files.storage import default_storage from django.core.validators import URLValidator @@ -62,6 +62,10 @@ VALID_FILE_EXTENSIONS = [".xlsx", ".csv"] +# pylint: disable=invalid-name +User = get_user_model() + + def get_filename(response): """ Get filename from a Content-Desposition header. @@ -71,9 +75,9 @@ def get_filename(response): # following format: # 'attachment; filename="ActApp_Survey_System.xlsx"; filename*=UTF-8\'\'ActApp_Survey_System.xlsx' # noqa cleaned_xls_file = "" - content = response.headers.get("content-disposition").split("; ") + content = response.headers.get("Content-Disposition").split("; ") counter = [a for a in content if a.startswith("filename=")] - if len(counter) >= 1: + if counter: filename_key_val = counter[0] filename = filename_key_val.split("=")[1].replace('"', "") name, extension = os.path.splitext(filename) @@ -124,7 +128,7 @@ class PermissionForm(forms.Form): def __init__(self, username): self.username = username - super(PermissionForm, self).__init__() + super().__init__() class UserProfileForm(ModelForm): @@ -134,6 +138,7 @@ class UserProfileForm(ModelForm): class Meta: model = UserProfile + # pylint: disable=modelform-uses-exclude exclude = ("user", "created_by", "num_of_submissions") email = forms.EmailField(widget=forms.TextInput()) @@ -144,7 +149,7 @@ def clean_metadata(self): """ metadata = self.cleaned_data.get("metadata") - return metadata if metadata is not None else dict() + return metadata if metadata is not None else {} class UserProfileFormRegister(forms.Form): @@ -207,14 +212,11 @@ def clean_username(self): if username in self.RESERVED_USERNAMES: raise forms.ValidationError( - _("%s is a reserved name, please choose another") % username + _(f"{username} is a reserved name, please choose another") ) - elif not self.legal_usernames_re.search(username): + if not self.legal_usernames_re.search(username): raise forms.ValidationError( - _( - "username may only contain alpha-numeric characters and " - "underscores" - ) + _("username may only contain alpha-numeric characters and underscores") ) try: User.objects.get(username=username) @@ -356,13 +358,14 @@ def clean_project(self): project = self.cleaned_data["project"] if project is not None: try: - # pylint: disable=attribute-defined-outside-init, no-member + # pylint: disable=attribute-defined-outside-init,no-member self._project = Project.objects.get(pk=int(project)) - except (Project.DoesNotExist, ValueError): - raise forms.ValidationError(_("Unknown project id: %s" % project)) + except (Project.DoesNotExist, ValueError) as e: + raise forms.ValidationError(_(f"Unknown project id: {project}")) from e return project + # pylint: disable=too-many-locals def publish(self, user, id_string=None, created_by=None): """ Publish XLSForm. @@ -379,11 +382,11 @@ def publish(self, user, id_string=None, created_by=None): csv_data = self.cleaned_data["text_xls_form"] # assigning the filename to a random string (quick fix) - import random - rand_name = "uploaded_form_%s.csv" % "".join( + random_string = "".join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) ) + rand_name = f"uploaded_form_{random_string}.csv" cleaned_xls_file = default_storage.save( upload_to(None, rand_name, user.username), @@ -401,12 +404,13 @@ def publish(self, user, id_string=None, created_by=None): ) if cleaned_url: + self.validate(cleaned_url) cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = "_".join(cleaned_xls_file.path.split("/")[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: - response = requests.get(cleaned_url) + response = requests.head(cleaned_url) if ( response.headers.get("content-type") in VALID_XLSFORM_CONTENT_TYPES @@ -415,9 +419,10 @@ def publish(self, user, id_string=None, created_by=None): cleaned_xls_file = get_filename(response) cleaned_xls_file = upload_to(None, cleaned_xls_file, user.username) - self.validate(cleaned_url) - xls_data = ContentFile(urlopen(cleaned_url).read()) - cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) + response = requests.get(cleaned_url) + if response.status_code < 400: + xls_data = ContentFile(response.content) + cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data["project"] @@ -444,6 +449,7 @@ def publish(self, user, id_string=None, created_by=None): return publish_xls_form( cleaned_xls_file, user, project, id_string, created_by or user ) + return None class ActivateSMSSupportForm(forms.Form): diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index 6f2a01b929..9b8e9d0271 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -192,12 +192,13 @@ def validate_email_subject(self, value): """ Validate the email subject is not empty. """ - if len(value) == 0: + if value: return None return value def create(self, validated_data): + """Reset a user password.""" instance = PasswordReset(**validated_data) instance.save() @@ -223,11 +224,12 @@ def validate_uid(self, value): return value def validate(self, attrs): + """Validates the generated user token.""" user = get_user_from_uid(attrs.get("uid")) token = attrs.get("token") if not default_token_generator.check_token(user, token): - raise serializers.ValidationError(_("Invalid token: %s") % token) + raise serializers.ValidationError(_(f"Invalid token: {token}")) return attrs diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index cf918952ea..9f30315913 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -626,11 +626,7 @@ def get_ordered_repeat_value(xpath, repeat_value): # check for lists # pylint: disable=too-many-nested-blocks - if ( - isinstance(value, list) - and len(value) > 0 - and key not in [ATTACHMENTS, NOTES] - ): + if isinstance(value, list) and value and key not in [ATTACHMENTS, NOTES]: for index, item in enumerate(value): # start at 1 index += 1 @@ -829,6 +825,7 @@ def _format_for_dataframe(self, cursor): yield flat_dict def export_to(self, path, dataview=None): + """Export a CSV formated to the given ``path``.""" self.ordered_columns = OrderedDict() self._build_ordered_columns(self.data_dictionary.survey, self.ordered_columns) From a471feef3b603b2512e1364cf84288cfb8db5b64 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 3 May 2022 22:01:16 +0300 Subject: [PATCH 103/234] Fix: urlopen tests to use requests mocked response. --- .../api/tests/viewsets/test_xform_viewset.py | 230 ++++++++++-------- onadata/apps/logger/models/xform.py | 111 ++++++--- onadata/apps/main/tests/test_process.py | 39 +-- 3 files changed, 236 insertions(+), 144 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index f8e0399099..cd483bce17 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -19,6 +19,7 @@ from xml.dom import minidom import jwt +import requests from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.cache import cache @@ -90,7 +91,14 @@ ) +ROLES = [ReadOnlyRole, DataEntryRole, EditorRole, ManagerRole, OwnerRole] + +JWT_SECRET_KEY = "thesecretkey" +JWT_ALGORITHM = "HS256" + + def fixtures_path(filepath): + """Returns the file object at the given filepath.""" return open( os.path.join( settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", filepath @@ -106,10 +114,24 @@ def raise_bad_status_line(arg): raise BadStatusLine("RANDOM STATUS") -ROLES = [ReadOnlyRole, DataEntryRole, EditorRole, ManagerRole, OwnerRole] +# pylint: disable=invalid-name +def get_mocked_response_for_file(file_object, filename, status_code=200): + """Returns a requests.Response() object for mocked tests.""" + mock_response = requests.Response() + mock_response.status_code = status_code + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" + ), + "Content-Disposition": ( + 'attachment; filename="transportation.' + f"xlsx\"; filename*=UTF-8''{filename}" + ), + } + # pylint: disable=protected-access + mock_response._content = file_object.read() -JWT_SECRET_KEY = "thesecretkey" -JWT_ALGORITHM = "HS256" + return mock_response class TestXFormViewSet(TestAbstractViewSet): @@ -1355,8 +1377,9 @@ def test_publish_xlsforms_with_same_id_string(self): self.assertIsInstance(xform, XForm) self.assertEqual(counter + 2, XForm.objects.count()) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_xlsform_using_url_upload(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1372,21 +1395,26 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - mock_urlopen.assert_called_with(xls_url) - xls_file.close() + mock_requests.get.assert_called_with(xls_url) + xls_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_xlsform_using_url_with_no_extension(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_with_no_extension(self, mock_requests): with HTTMock(enketo_mock, xls_url_no_extension_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1402,19 +1430,24 @@ def test_publish_xlsform_using_url_with_no_extension(self, mock_urlopen): "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201, response.data) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v1( - self, mock_urlopen + self, mock_requests ): with HTTMock( enketo_mock, xls_url_no_extension_mock_content_disposition_attr_jumbled_v1 @@ -1433,19 +1466,24 @@ def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v1( "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v2( - self, mock_urlopen + self, mock_requests ): with HTTMock( enketo_mock, xls_url_no_extension_mock_content_disposition_attr_jumbled_v2 @@ -1464,18 +1502,23 @@ def test_publish_xlsform_using_url_content_disposition_attr_jumbled_v2( "transportation_different_id_string.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"xls_url": xls_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") - def test_publish_csvform_using_url_upload(self, mock_urlopen): + # pylint: disable=invalid-name + @patch("onadata.apps.main.forms.requests") + def test_publish_csvform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -1490,19 +1533,24 @@ def test_publish_csvform_using_url_upload(self, mock_urlopen): "text_and_integer.csv", ) - csv_file = open(path, "rb") - mock_urlopen.return_value = csv_file + with open(path, "rb") as csv_file: + mock_response = get_mocked_response_for_file( + csv_file, "text_and_integer.csv", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"csv_url": csv_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) + post_data = {"csv_url": csv_url} + request = self.factory.post("/", data=post_data, **self.extra) + response = view(request) - mock_urlopen.assert_called_with(csv_url) - csv_file.close() + mock_requests.get.assert_called_with(csv_url) + csv_file.close() - self.assertEqual(response.status_code, 201) - self.assertEqual(XForm.objects.count(), pre_count + 1) + self.assertEqual(response.status_code, 201) + self.assertEqual(XForm.objects.count(), pre_count + 1) + # pylint: disable=invalid-name def test_publish_select_external_xlsform(self): with HTTMock(enketo_urls_mock): view = XFormViewSet.as_view({"post": "create"}) @@ -2887,8 +2935,8 @@ def test_update_xform_xls_file_with_version_set(self): self.assertEqual(self.xform.version, "212121211") self.assertEqual(form_id, self.xform.pk) - @patch("onadata.apps.main.forms.urlopen") - def test_update_xform_xls_url(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_update_xform_xls_url(self, mock_requests): with HTTMock(enketo_mock): self._publish_xls_form_to_project() form_id = self.xform.pk @@ -2913,24 +2961,28 @@ def test_update_xform_xls_url(self, mock_urlopen): "transportation_version.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"xls_url": xls_url} - request = self.factory.patch("/", data=post_data, **self.extra) - response = view(request, pk=form_id) + post_data = {"xls_url": xls_url} + request = self.factory.patch("/", data=post_data, **self.extra) + response = view(request, pk=form_id) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.xform.refresh_from_db() + self.xform.refresh_from_db() - self.assertEqual(count, XForm.objects.all().count()) - # diff versions - self.assertEqual(self.xform.version, "212121211") - self.assertEqual(form_id, self.xform.pk) + self.assertEqual(count, XForm.objects.all().count()) + # diff versions + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) - @patch("onadata.apps.main.forms.urlopen") - def test_update_xform_dropbox_url(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_update_xform_dropbox_url(self, mock_requests): with HTTMock(enketo_mock): self._publish_xls_form_to_project() form_id = self.xform.pk @@ -2955,21 +3007,25 @@ def test_update_xform_dropbox_url(self, mock_urlopen): "transportation_version.xlsx", ) - xls_file = open(path, "rb") - mock_urlopen.return_value = xls_file + with open(path, "rb") as xls_file: + mock_response = get_mocked_response_for_file( + xls_file, "transportation_version.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response - post_data = {"dropbox_xls_url": xls_url} - request = self.factory.patch("/", data=post_data, **self.extra) - response = view(request, pk=form_id) + post_data = {"dropbox_xls_url": xls_url} + request = self.factory.patch("/", data=post_data, **self.extra) + response = view(request, pk=form_id) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) - self.xform.refresh_from_db() + self.xform.refresh_from_db() - self.assertEqual(count, XForm.objects.all().count()) - # diff versions - self.assertEqual(self.xform.version, "212121211") - self.assertEqual(form_id, self.xform.pk) + self.assertEqual(count, XForm.objects.all().count()) + # diff versions + self.assertEqual(self.xform.version, "212121211") + self.assertEqual(form_id, self.xform.pk) def test_update_xform_using_put_with_invalid_input(self): with HTTMock(enketo_mock): @@ -5421,24 +5477,6 @@ def test_csv_xls_import_errors(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.data.get("error"), "csv_file not a csv file") - @patch("onadata.apps.main.forms.urlopen", side_effect=raise_bad_status_line) - def test_error_raised_xform_url_upload_urllib_error(self, mock_urlopen): - """ - Test that the BadStatusLine error thrown by urlopen when a status - code is not understood is handled properly - """ - view = XFormViewSet.as_view({"post": "create"}) - - xls_url = "http://localhost:2000" - - post_data = {"xls_url": xls_url} - request = self.factory.post("/", data=post_data, **self.extra) - response = view(request) - error_msg = "An error occurred while publishing the form. " "Please try again." - - self.assertEqual(response.status_code, 400) - self.assertEqual(response.data.get("text"), error_msg) - def test_export_csvzip_form_data_async(self): with HTTMock(enketo_mock): xls_path = os.path.join( diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 987401268a..0a4f14e939 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -3,12 +3,11 @@ The XForm model """ # pylint: disable=too-many-lines +import hashlib import json import os import re -from builtins import bytes as b, str as text from datetime import datetime -from hashlib import md5 from xml.dom import Node import pytz @@ -71,20 +70,24 @@ "note", ] XFORM_TITLE_LENGTH = 255 -title_pattern = re.compile(r"(.*?)") +TITLE_PATTERN = re.compile(r"(.*?)") +# pylint: disable=invalid-name User = get_user_model() def cmp(x, y): + """Returns the difference on the comparison of ``x`` and ``y``.""" return (x > y) - (x < y) def question_types_to_exclude(_type): + """Returns True if ``_type`` is in QUESTION_TYPES_TO_EXCLUDE.""" return _type in QUESTION_TYPES_TO_EXCLUDE def upload_to(instance, filename): + """Returns the path to upload an XLSForm file to.""" return os.path.join(instance.user.username, "xls", os.path.split(filename)[1]) @@ -94,8 +97,22 @@ def contains_xml_invalid_char(text, invalids=None): return 1 in [c in text for c in invalids] +def _additional_headers(): + return [ + "_xform_id_string", + "_percentage_complete", + "_status", + "_attachments", + "_potential_duplicates", + ] + + class DictOrganizer: + """Adds parent index information in a submission record.""" + def set_dict_iterator(self, dict_iterator): + """Set's the dict iterator.""" + # pylint: disable=attribute-defined-outside-init self._dict_iterator = dict_iterator # Every section will get its own table @@ -144,6 +161,7 @@ def _build_obs_from_dict( return obs def get_observation_from_dict(self, item): + """Returns a dict that has observations added from ``item``.""" if len(list(item)) != 1: raise AssertionError() root_name = list(item)[0] @@ -159,7 +177,7 @@ def get_observation_from_dict(self, item): class DuplicateUUIDError(Exception): - pass + """Exception to raise when there are duplicate UUIDS in an XForm XML.""" def get_forms_shared_with_user(user): @@ -204,6 +222,7 @@ def _expand_select_all_that_apply(item, key, elem): del item[key] +# pylint: disable=too-many-instance-attributes,too-many-public-methods class XFormMixin: """XForm mixin class - adds helper functions.""" @@ -275,9 +294,9 @@ def _set_uuid_in_xml(self, file_name=None): if node.nodeType == Node.ELEMENT_NODE and node.tagName == "uuid" ] - if len(uuid_nodes) == 0: + if not uuid_nodes: formhub_node.appendChild(doc.createElement("uuid")) - if len(formhub_nodes) == 0: + if not formhub_nodes: # append the calculate bind node calculate_node = doc.createElement("bind") calculate_node.setAttribute( @@ -302,15 +321,20 @@ def _set_uuid_in_xml(self, file_name=None): ) self.xml = inline_output + # pylint: disable=too-few-public-methods class Meta: + """A proxy Meta class""" + app_label = "viewer" proxy = True @property def has_id_string_changed(self): + """Returns the boolean value of `_id_string_changed`.""" return getattr(self, "_id_string_changed", False) def add_instances(self): + """Returns all instances as a list of python objects.""" _get_observation_from_dict = DictOrganizer().get_observation_from_dict return [ @@ -327,13 +351,14 @@ def _id_string_already_exists_in_account(self, id_string): return True def get_unique_id_string(self, id_string, count=0): + """Checks and returns a unique ``id_string``.""" # used to generate a new id_string for new data_dictionary object if # id_string already existed if self._id_string_already_exists_in_account(id_string): if count != 0: if re.match(r"\w+_\d+$", id_string): - a = id_string.split("_") - id_string = "_".join(a[:-1]) + parts = id_string.split("_") + id_string = "_".join(parts[:-1]) count += 1 id_string = f"{id_string}_{count}" @@ -342,6 +367,7 @@ def get_unique_id_string(self, id_string, count=0): return id_string def get_survey(self): + """Returns an XML XForm survey object.""" if not hasattr(self, "_survey"): try: builder = SurveyElementBuilder() @@ -350,13 +376,14 @@ def get_survey(self): if isinstance(self.json, dict): self._survey = builder.create_survey_element_from_dict(self.json) except ValueError: - xml = b(bytearray(self.xml, encoding="utf-8")) + xml = bytes(bytearray(self.xml, encoding="utf-8")) self._survey = create_survey_element_from_xml(xml) return self._survey survey = property(get_survey) def get_survey_elements(self): + """Returns an iterator of all survey elements.""" return self.survey.iter_descendants() def get_survey_element(self, name_or_xpath): @@ -374,7 +401,7 @@ def get_survey_element(self, name_or_xpath): field for field in self.get_survey_elements() if field.name == name_or_xpath ] - return fields[0] if len(fields) else None + return fields[0] if fields else None def get_child_elements(self, name_or_xpath, split_select_multiples=True): """Returns a list of survey elements children in a flat list. @@ -382,16 +409,16 @@ def get_child_elements(self, name_or_xpath, split_select_multiples=True): appended to the list. If the name_or_xpath is a repeat we iterate through the child elements as well. """ - GROUP_AND_SELECT_MULTIPLES = ["group"] + group_and_select_multiples = ["group"] if split_select_multiples: - GROUP_AND_SELECT_MULTIPLES += ["select all that apply"] + group_and_select_multiples += ["select all that apply"] def flatten(elem, items=None): items = [] if items is None else items results = [] if elem: xpath = elem.get_abbreviated_xpath() - if elem.type in GROUP_AND_SELECT_MULTIPLES or ( + if elem.type in group_and_select_multiples or ( xpath == name_or_xpath and elem.type == "repeat" ): for child in elem.children: @@ -405,9 +432,11 @@ def flatten(elem, items=None): return flatten(element) + # pylint: disable=no-self-use def get_choice_label(self, field, choice_value, lang="English"): + """Returns a choice's label for the given ``field`` and ``choice_value``.""" choices = [choice for choice in field.children if choice.name == choice_value] - if len(choices): + if choices: choice = choices[0] label = choice.label @@ -426,13 +455,14 @@ def get_mongo_field_names_dict(self): names = {} for elem in self.get_survey_elements(): names[ - _encode_for_mongo(text(elem.get_abbreviated_xpath())) + _encode_for_mongo(str(elem.get_abbreviated_xpath())) ] = elem.get_abbreviated_xpath() return names survey_elements = property(get_survey_elements) def get_field_name_xpaths_only(self): + """Returns the abbreviated_xpath of all fields in a survey form.""" return [ elem.get_abbreviated_xpath() for elem in self.survey_elements @@ -440,6 +470,7 @@ def get_field_name_xpaths_only(self): ] def geopoint_xpaths(self): + """Returns the abbreviated_xpath of all fields of type `geopoint`.""" survey_elements = self.get_survey_elements() return [ @@ -449,6 +480,7 @@ def geopoint_xpaths(self): ] def xpath_of_first_geopoint(self): + """Returns the abbreviated_xpath of the first field of type `geopoint`.""" geo_xpaths = self.geopoint_xpaths() return len(geo_xpaths) and geo_xpaths[0] @@ -464,7 +496,7 @@ def xpaths(self, prefix="", survey_element=None, result=None, repeat_iterations= return [] result = [] if result is None else result - path = "/".join([prefix, text(survey_element.name)]) + path = "/".join([prefix, str(survey_element.name)]) if survey_element.children is not None: # add xpaths to result for each child @@ -511,21 +543,13 @@ def get_additional_geopoint_xpaths(cls, xpath): return ["_".join([prefix, name, suffix]) for suffix in cls.GEODATA_SUFFIXES] - def _additional_headers(self): - return [ - "_xform_id_string", - "_percentage_complete", - "_status", - "_attachments", - "_potential_duplicates", - ] - def get_headers(self, include_additional_headers=False, repeat_iterations=4): """ Return a list of headers for a csv file. """ def shorten(xpath): + """Returns the shortened part of an ``xpath`` removing the root node.""" xpath_list = xpath.split("/") return "/".join(xpath_list[2:]) @@ -549,13 +573,14 @@ def shorten(xpath): MEDIA_ALL_RECEIVED, ] if include_additional_headers: - header_list += self._additional_headers() + header_list += _additional_headers() return header_list def get_keys(self): """Return all XForm headers.""" def remove_first_index(xpath): + """Removes the first index from an ``xpath``.""" return re.sub(r"\[1\]", "", xpath) return [remove_first_index(header) for header in self.get_headers()] @@ -568,6 +593,7 @@ def get_element(self, abbreviated_xpath): self._survey_elements[e.get_abbreviated_xpath()] = e def remove_all_indices(xpath): + """Removes all indices from an ``xpath``.""" return re.sub(r"\[\d+\]", "", xpath) clean_xpath = remove_all_indices(abbreviated_xpath) @@ -585,7 +611,7 @@ def get_default_language(self): def get_language(self, languages, language_index=0): """Returns the language at the given index.""" language = None - if isinstance(languages, list) and len(languages): + if isinstance(languages, list) and languages: if self.default_language in languages: language_index = languages.index(self.default_language) @@ -615,6 +641,7 @@ def get_xpath_cmp(self): if not hasattr(self, "_xpaths"): self._xpaths = [e.get_abbreviated_xpath() for e in self.survey_elements] + # pylint: disable=invalid-name def xpath_cmp(x, y): # For the moment, we aren't going to worry about repeating # nodes. @@ -660,8 +687,8 @@ def get_variable_name(self, abbreviated_xpath): return header def get_list_of_parsed_instances(self, flat=True): + """Return an iterator of all parsed instances.""" for i in queryset_iterator(self.instances_for_export(self)): - # TODO: there is information we want to add in parsed xforms. yield i.get_dict(flat=flat) def _rename_key(self, item, old_key, new_key): @@ -698,7 +725,9 @@ def _mark_start_time_boolean(self): def get_survey_elements_of_type(self, element_type): return [e for e in self.get_survey_elements() if e.type == element_type] + # pylint: disable=invalid-name def get_survey_elements_with_choices(self): + """Returns all survey elements of type SELECT_ONE and SELECT_ALL_THAT_APPLY.""" if not hasattr(self, "_survey_elements_with_choices"): choices_type = [constants.SELECT_ONE, constants.SELECT_ALL_THAT_APPLY] @@ -746,6 +775,7 @@ def get_select_multiple_xpaths(self): return self._select_multiple_xpaths def get_media_survey_xpaths(self): + """Returns all survey element abbreviated_xpath of type in KNOWN_MEDIA_TYPES""" return [ e.get_abbreviated_xpath() for e in sum( @@ -763,11 +793,15 @@ def get_osm_survey_xpaths(self): ] +# pylint: disable=too-many-instance-attributes class XForm(XFormMixin, BaseModel): + """XForm model - stores the XLSForm and related data.""" + CLONED_SUFFIX = "_cloned" MAX_ID_LENGTH = 100 xls = models.FileField(upload_to=upload_to, null=True) + # pylint: disable=no-member json = models.JSONField(default=dict) description = models.TextField(default="", null=True, blank=True) xml = models.TextField() @@ -857,9 +891,11 @@ class Meta: ) def file_name(self): + """Returns the XML filename based on the ``self.id_string``.""" return self.id_string + ".xml" def url(self): + """Returns the download URL for the XForm.""" return reverse( "download_xform", kwargs={"username": self.user.username, "id_string": self.id_string}, @@ -867,6 +903,7 @@ def url(self): @property def has_instances_with_geopoints(self): + """Returns instances with geopoints.""" return self.instances_with_geopoints def _set_id_string(self): @@ -877,7 +914,7 @@ def _set_id_string(self): def _set_title(self): xml = re.sub(r"\s+", " ", self.xml) - matches = title_pattern.findall(xml) + matches = TITLE_PATTERN.findall(xml) if len(matches) != 1: raise XLSFormError(_("There should be a single title."), matches) @@ -889,9 +926,9 @@ def _set_title(self): if self.title and title_xml != self.title: title_xml = self.title[:XFORM_TITLE_LENGTH] - if isinstance(self.xml, b): + if isinstance(self.xml, bytes): self.xml = self.xml.decode("utf-8") - self.xml = title_pattern.sub(f"{title_xml}", self.xml) + self.xml = TITLE_PATTERN.sub(f"{title_xml}", self.xml) self._set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( @@ -929,7 +966,7 @@ def update(self, *args, **kwargs): """Persists the form to the DB.""" super().save(*args, **kwargs) - # pylint: disable=too-many-branches + # pylint: disable=too-many-branches,arguments-differ def save(self, *args, **kwargs): # noqa: MC0001 """Sets additional form properties before saving to the DB""" update_fields = kwargs.get("update_fields") @@ -984,7 +1021,7 @@ def save(self, *args, **kwargs): # noqa: MC0001 self.sms_id_string = json.loads(self.json).get( "sms_keyword", self.id_string ) - except Exception: + except ValueError: self.sms_id_string = self.id_string if update_fields is None or "public_key" in update_fields: @@ -1088,6 +1125,7 @@ def submission_count(self, force_update=False): @property def submission_count_for_today(self): + """Returns the submissions count for the current day.""" current_timzone_name = timezone.get_current_timezone_name() current_timezone = pytz.timezone(current_timzone_name) today = datetime.today() @@ -1135,7 +1173,10 @@ def time_of_last_submission_update(self): def get_hash(self): """Returns the MD5 hash of the forms XML content prefixed by 'md5:'""" - return f"md5:{md5(self.xml.encode('utf8')).hexdigest()}" + md5_hash = hashlib.new( + "md5", self.xml.encode("utf-8"), usedforsecurity=False + ).hexdigest() + return f"md5:{md5_hash}" @property def can_be_replaced(self): @@ -1201,12 +1242,14 @@ def xform_post_delete_callback(sender, instance, **kwargs): ) +# pylint: disable=too-few-public-methods class XFormUserObjectPermission(UserObjectPermissionBase): """Guardian model to create direct foreign keys.""" content_object = models.ForeignKey(XForm, on_delete=models.CASCADE) +# pylint: disable=too-few-public-methods class XFormGroupObjectPermission(GroupObjectPermissionBase): """Guardian model to create direct foreign keys.""" diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 0c48d651e8..3cbee927b5 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -24,7 +24,7 @@ from six import iteritems from onadata.apps.logger.models import XForm -from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH +from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH, _additional_headers from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase @@ -101,8 +101,7 @@ def test_publish_xlsx_file(self): self._publish_xlsx_file() @patch("onadata.apps.main.forms.requests") - @patch("onadata.apps.main.forms.urlopen") - def test_google_url_upload(self, mock_urlopen, mock_requests): + def test_google_url_upload(self, mock_requests): """Test uploading an XLSForm from a Google Docs SpreadSheet URL.""" if self._internet_on(url="http://google.com"): xls_url = ( @@ -129,25 +128,26 @@ def test_google_url_upload(self, mock_urlopen, mock_requests): "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" ), - "content-disposition": ( + "Content-Disposition": ( 'attachment; filename="transportation.' "xlsx\"; filename*=UTF-8''transportation.xlsx" ), } + mock_requests.head.return_value = mock_response + # pylint: disable=protected-access + mock_response._content = xls_file.read() mock_requests.get.return_value = mock_response - mock_urlopen.return_value = xls_file response = self.client.post( f"/{self.user.username}/", {"xls_url": xls_url} ) - mock_urlopen.assert_called_with(xls_url) mock_requests.get.assert_called_with(xls_url) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) - @patch("onadata.apps.main.forms.urlopen") - def test_url_upload(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_url_upload(self, mock_requests): """Test uploading an XLSForm from a URL.""" if self._internet_on(url="http://google.com"): xls_url = "https://ona.io/examples/forms/tutorial/form.xlsx" @@ -165,13 +165,26 @@ def test_url_upload(self, mock_urlopen): # pylint: disable=consider-using-with with open(path, "rb") as xls_file: - mock_urlopen.return_value = xls_file + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" + "officedocument.spreadsheetml.sheet" + ), + "content-disposition": ( + 'attachment; filename="transportation.' + "xlsx\"; filename*=UTF-8''transportation.xlsx" + ), + } + # pylint: disable=protected-access + mock_response._content = xls_file.read() + mock_requests.get.return_value = mock_response response = self.client.post( f"/{self.user.username}/", {"xls_url": xls_url} ) - - mock_urlopen.assert_called_with(xls_url) + mock_requests.get.assert_called_with(xls_url) # make sure publishing the survey worked self.assertEqual(response.status_code, 200) @@ -506,9 +519,7 @@ def _check_csv_export_second_pass(self): }, ] - data_dictionary = DataDictionary.objects.get(pk=self.xform.pk) - # pylint: disable=protected-access - additional_headers = data_dictionary._additional_headers() + [ + additional_headers = _additional_headers() + [ "_id", "_date_modified", ] From 5fc0e66cd9b2de65e79d1c295fe57cc8f0edfec0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 3 May 2022 22:53:12 +0300 Subject: [PATCH 104/234] Fixes: urlopen removal in test and setting xform.sms_id_string --- .../tests/viewsets/test_abstract_viewset.py | 22 +++++++++++++++ .../tests/viewsets/test_project_viewset.py | 17 ++++++++---- .../api/tests/viewsets/test_xform_viewset.py | 27 +++++-------------- onadata/apps/logger/models/xform.py | 9 +++---- 4 files changed, 43 insertions(+), 32 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 13ec6a3641..c958c7a54d 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -7,6 +7,8 @@ import re from tempfile import NamedTemporaryFile +import requests + from django.conf import settings from django.contrib.auth import authenticate, get_user_model from django.contrib.auth.models import Permission @@ -70,6 +72,26 @@ def add_uuid_to_submission_xml(path, xform): return path +# pylint: disable=invalid-name +def get_mocked_response_for_file(file_object, filename, status_code=200): + """Returns a requests.Response() object for mocked tests.""" + mock_response = requests.Response() + mock_response.status_code = status_code + mock_response.headers = { + "content-type": ( + "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" + ), + "Content-Disposition": ( + 'attachment; filename="transportation.' + f"xlsx\"; filename*=UTF-8''{filename}" + ), + } + # pylint: disable=protected-access + mock_response._content = file_object.read() + + return mock_response + + # pylint: disable=too-many-instance-attributes class TestAbstractViewSet(PyxformMarkdown, TestCase): """ diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index 63beebf533..e56fbcd73b 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -20,7 +20,10 @@ import requests from onadata.apps.api import tools -from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + get_mocked_response_for_file, +) from onadata.apps.api.tools import get_or_create_organization_owners_team from onadata.apps.api.viewsets.organization_profile_viewset import ( OrganizationProfileViewSet, @@ -93,8 +96,8 @@ def tearDown(self): super().tearDown() # pylint: disable=invalid-name - @patch("onadata.apps.main.forms.urlopen") - def test_publish_xlsform_using_url_upload(self, mock_urlopen): + @patch("onadata.apps.main.forms.requests") + def test_publish_xlsform_using_url_upload(self, mock_requests): with HTTMock(enketo_mock): self._project_create() view = ProjectViewSet.as_view({"post": "forms"}) @@ -113,13 +116,17 @@ def test_publish_xlsform_using_url_upload(self, mock_urlopen): ) with open(path, "rb") as xls_file: - mock_urlopen.return_value = xls_file + mock_response = get_mocked_response_for_file( + xls_file, "transportation_different_id_string.xlsx", 200 + ) + mock_requests.head.return_value = mock_response + mock_requests.get.return_value = mock_response post_data = {"xls_url": xls_url} request = self.factory.post("/", data=post_data, **self.extra) response = view(request, pk=project_id) - mock_urlopen.assert_called_with(xls_url) + mock_requests.get.assert_called_with(xls_url) xls_file.close() self.assertEqual(response.status_code, 201) self.assertEqual(XForm.objects.count(), pre_count + 1) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index cd483bce17..b28fb0e5fc 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -50,7 +50,10 @@ xls_url_no_extension_mock_content_disposition_attr_jumbled_v1, xls_url_no_extension_mock_content_disposition_attr_jumbled_v2, ) -from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet, + get_mocked_response_for_file, +) from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment @@ -114,27 +117,9 @@ def raise_bad_status_line(arg): raise BadStatusLine("RANDOM STATUS") -# pylint: disable=invalid-name -def get_mocked_response_for_file(file_object, filename, status_code=200): - """Returns a requests.Response() object for mocked tests.""" - mock_response = requests.Response() - mock_response.status_code = status_code - mock_response.headers = { - "content-type": ( - "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" - ), - "Content-Disposition": ( - 'attachment; filename="transportation.' - f"xlsx\"; filename*=UTF-8''{filename}" - ), - } - # pylint: disable=protected-access - mock_response._content = file_object.read() - - return mock_response - - class TestXFormViewSet(TestAbstractViewSet): + """Test XFormViewSet""" + def setUp(self): super(TestXFormViewSet, self).setUp() self.view = XFormViewSet.as_view( diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 0a4f14e939..bd6629818b 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1014,15 +1014,12 @@ def save(self, *args, **kwargs): # noqa: MC0001 if not self.sms_id_string and ( update_fields is None or "id_string" in update_fields ): - try: - # try to guess the form's wanted sms_id_string - # from it's json rep (from XLSForm) - # otherwise, use id_string to ensure uniqueness + if isinstance(self.json, str): self.sms_id_string = json.loads(self.json).get( "sms_keyword", self.id_string ) - except ValueError: - self.sms_id_string = self.id_string + else: + self.sms_id_string = self.json.get("sms_keyword", self.id_string) if update_fields is None or "public_key" in update_fields: self._set_public_key_field() From 60bb1fed8894705a8c4b856d6d0ce53844758b37 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 09:26:20 +0300 Subject: [PATCH 105/234] Remove unused import of requests --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b28fb0e5fc..5aad3c0c17 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -19,7 +19,6 @@ from xml.dom import minidom import jwt -import requests from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.cache import cache From fcff196e79322e1f4e05e4a85e0aea863401bbf4 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 4 May 2022 09:34:12 +0300 Subject: [PATCH 106/234] Handle stringified dicts Signed-off-by: Kipchirchir Sigei --- onadata/libs/utils/api_export_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 600dd55043..65e942adcc 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -576,7 +576,8 @@ def response_for_format(data, format=None): formatted_data = data.xls else: - formatted_data = json.loads(data.json) + formatted_data = json.loads(data.json) \ + if isinstance(data.json, str) else data.json return Response(formatted_data) From b5dc8de79db149b85c30c2a77ee27ad64ec550c2 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 4 May 2022 10:38:52 +0300 Subject: [PATCH 107/234] Use create_survey_element_from_(json|dict) where necessary Signed-off-by: Kipchirchir Sigei --- onadata/apps/logger/models/xform.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index bd6629818b..c42fa7d409 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -207,7 +207,12 @@ def check_version_set(survey): # set utc time as the default version survey_json["version"] = datetime.utcnow().strftime("%Y%m%d%H%M") builder = SurveyElementBuilder() - survey = builder.create_survey_element_from_json(json.dumps(survey_json)) + if isinstance(survey_json, str): + survey = builder.create_survey_element_from_json(survey_json) + elif isinstance(survey_json, dict): + survey = builder.create_survey_element_from_dict( + survey_json + ) return survey From 51f6a9cf4965b796e2b9ef5d5c103e3dbbdef137 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 4 May 2022 11:22:03 +0300 Subject: [PATCH 108/234] Build onadata ECR image with python 3.9 Signed-off-by: Kipchirchir Sigei --- docker/onadata-uwsgi/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index 5a4ea72037..6bfc2acd2e 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 as intermediate +FROM python:3.9 as intermediate ENV DEBIAN_FRONTEND noninteractive ENV PYTHONUNBUFFERED 1 @@ -14,7 +14,7 @@ RUN mkdir -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts RUN --mount=type=ssh if [ -n "$optional_packages" ]; then pip install ${optional_packages} ; fi FROM ubuntu:20.04 -COPY --from=intermediate /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/dist-packages/ +COPY --from=intermediate /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/dist-packages/ ARG release_version=v2.4.1 @@ -35,8 +35,8 @@ RUN apt-get update -q &&\ libmemcached-dev \ build-essential \ supervisor \ - python3.8 \ - python3.8-dev \ + python3.9 \ + python3.9-dev \ python3-pip \ python3-setuptools \ git \ @@ -47,6 +47,7 @@ RUN apt-get update -q &&\ libjpeg-dev \ libxml2-dev \ libxslt1-dev \ + libpython3.9-dev \ zlib1g-dev \ ghostscript \ python3-celery \ From 7959c5fc18e49b5cd5ee525328f2ecc48c3ac652 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 4 May 2022 12:28:35 +0300 Subject: [PATCH 109/234] Use python3-dev apt dependency Signed-off-by: Kipchirchir Sigei --- docker/onadata-uwsgi/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index 6bfc2acd2e..18a8ce07d8 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update -q &&\ build-essential \ supervisor \ python3.9 \ - python3.9-dev \ + python3-dev \ python3-pip \ python3-setuptools \ git \ From 47d335d8e1d7eca97772875378dd10060321dda0 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 4 May 2022 12:59:49 +0300 Subject: [PATCH 110/234] Instal pyyaml in docker builds Signed-off-by: Kipchirchir Sigei --- docker/onadata-uwsgi/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index 18a8ce07d8..a62d48b8e0 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -84,7 +84,7 @@ RUN python3 -m pip install --no-cache-dir -U pip && \ python3 -m pip install --no-cache-dir -r requirements/base.pip && \ python3 -m pip install --no-cache-dir -r requirements/s3.pip && \ python3 -m pip install --no-cache-dir -r requirements/ses.pip && \ - python3 -m pip install --no-cache-dir uwsgitop django-prometheus==v2.2.0 + python3 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0 # Compile API Docs RUN make -C docs html From a4faaef1c97c164b077e27032cb322c03141fa96 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 21:21:46 +0300 Subject: [PATCH 111/234] main.views: cleanup --- onadata/apps/main/views.py | 1757 +++++++++++++++++++----------------- 1 file changed, 922 insertions(+), 835 deletions(-) diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 418d4dc71a..ed24267772 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -6,25 +6,31 @@ import json import os from datetime import datetime +from http import HTTPStatus from bson import json_util +from bson.objectid import ObjectId from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.core.files.storage import default_storage, get_storage_class -from django.db import IntegrityError, OperationalError -from django.http import (HttpResponse, HttpResponseBadRequest, - HttpResponseForbidden, HttpResponseNotFound, - HttpResponseRedirect, HttpResponseServerError) +from django.db import IntegrityError +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) from django.shortcuts import get_object_or_404, render -from django.template import RequestContext, loader +from django.template import loader from django.urls import reverse from django.utils.translation import ugettext as _ from django.utils.html import conditional_escape -from django.views.decorators.http import (require_GET, require_http_methods, - require_POST) +from django.views.decorators.http import require_GET, require_http_methods, require_POST from guardian.shortcuts import assign_perm, remove_perm from oauth2_provider.views.base import AuthorizationView from rest_framework.authtoken.models import Token @@ -33,54 +39,73 @@ from onadata.apps.logger.models.xform import get_forms_shared_with_user from onadata.apps.logger.views import enter_data from onadata.apps.logger.xform_instance_parser import XLSFormError -from onadata.apps.main.forms import (ActivateSMSSupportForm, DataLicenseForm, - ExternalExportForm, FormLicenseForm, - MapboxLayerForm, MediaForm, - PermissionForm, QuickConverter, - QuickConverterFile, QuickConverterURL, - SourceForm, SupportDocForm, - UserProfileForm) +from onadata.apps.main.forms import ( + ActivateSMSSupportForm, + DataLicenseForm, + ExternalExportForm, + FormLicenseForm, + MapboxLayerForm, + MediaForm, + PermissionForm, + QuickConverter, + QuickConverterFile, + QuickConverterURL, + SourceForm, + SupportDocForm, + UserProfileForm, +) from onadata.apps.main.models import AuditLog, MetaData, UserProfile from onadata.apps.sms_support.autodoc import get_autodoc_for from onadata.apps.sms_support.providers import providers_doc -from onadata.apps.sms_support.tools import (check_form_sms_compatibility, - is_sms_related) -from onadata.apps.viewer.models.data_dictionary import (DataDictionary, - upload_to) -from onadata.apps.viewer.models.parsed_instance import (DATETIME_FORMAT, - query_data) +from onadata.apps.sms_support.tools import check_form_sms_compatibility, is_sms_related +from onadata.apps.viewer.models.data_dictionary import DataDictionary, upload_to +from onadata.apps.viewer.models.parsed_instance import DATETIME_FORMAT, query_data from onadata.apps.viewer.views import attachment_url from onadata.libs.exceptions import EnketoError from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.export_tools import upload_template_for_external_export from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.logger_tools import (publish_form, - response_with_mimetype_and_name) +from onadata.libs.utils.logger_tools import ( + publish_form, + response_with_mimetype_and_name, +) from onadata.libs.utils.qrcode import generate_qrcode -from onadata.libs.utils.user_auth import (add_cors_headers, check_and_set_user, - check_and_set_user_and_form, - get_user_default_project, - get_xform_and_perms, - get_xform_users_with_perms, - has_permission, - helper_auth_helper, set_profile_data) -from onadata.libs.utils.viewer_tools import (get_enketo_urls, get_form) +from onadata.libs.utils.user_auth import ( + add_cors_headers, + check_and_set_user, + check_and_set_user_and_form, + get_user_default_project, + get_xform_and_perms, + get_xform_users_with_perms, + has_permission, + helper_auth_helper, + set_profile_data, +) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form + +# pylint: disable=invalid-name +User = get_user_model() def home(request): + """Default landing view.""" if request.user.username: return HttpResponseRedirect( - reverse(profile, kwargs={'username': request.user.username})) + reverse(profile, kwargs={"username": request.user.username}) + ) - return render(request, 'home.html') + return render(request, "home.html") @login_required def login_redirect(request): + """Redirects a user to their profile page on successful login.""" return HttpResponseRedirect( - reverse(profile, kwargs={'username': request.user.username})) + reverse(profile, kwargs={"username": request.user.username}) + ) +# pylint: disable=unused-argument @require_POST @login_required def clone_xlsform(request, username): @@ -89,59 +114,66 @@ def clone_xlsform(request, username): Eliminates the need to download Excel File and upload again. """ to_username = request.user.username - message = {'type': None, 'text': '....'} + message = {"type": None, "text": "...."} message_list = [] def set_form(): - form_owner = request.POST.get('username') - id_string = request.POST.get('id_string') - xform = XForm.objects.get(user__username__iexact=form_owner, - id_string__iexact=id_string, - deleted_at__isnull=True) + """Publishes the XLSForm creating a DataDictionary object.""" + form_owner = request.POST.get("username") + id_string = request.POST.get("id_string") + xform = XForm.objects.get( + user__username__iexact=form_owner, + id_string__iexact=id_string, + deleted_at__isnull=True, + ) if len(id_string) > 0 and id_string[0].isdigit(): - id_string = '_' + id_string + id_string = "_" + id_string path = xform.xls.name if default_storage.exists(path): project = get_user_default_project(request.user) - xls_file = upload_to(None, '%s%s.xlsx' % ( - id_string, XForm.CLONED_SUFFIX), to_username) + xls_file = upload_to( + None, f"{id_string}{XForm.CLONED_SUFFIX}.xlsx", to_username + ) xls_data = default_storage.open(path) xls_file = default_storage.save(xls_file, xls_data) survey = DataDictionary.objects.create( - user=request.user, - xls=xls_file, - project=project + user=request.user, xls=xls_file, project=project ).survey # log to cloner's account audit = {} audit_log( - Actions.FORM_CLONED, request.user, request.user, - _("Cloned form '%(id_string)s'.") % - { - 'id_string': survey.id_string, - }, audit, request) + Actions.FORM_CLONED, + request.user, + request.user, + _(f"Cloned form '{survey.id_string}'."), + audit, + request, + ) clone_form_url = reverse( - show, kwargs={ - 'username': to_username, - 'id_string': xform.id_string + XForm.CLONED_SUFFIX}) + show, + kwargs={ + "username": to_username, + "id_string": xform.id_string + XForm.CLONED_SUFFIX, + }, + ) + profile_url = reverse(profile, kwargs={"username": to_username}) + profile_url_link = f'profile.' + form_url_link = f'{survey.id_string} ' return { - 'type': 'alert-success', - 'text': _( - u'Successfully cloned to %(form_url)s into your ' - u'%(profile_url)s') % { - 'form_url': u'%(id_string)s ' % { - 'id_string': survey.id_string, - 'url': clone_form_url - }, - 'profile_url': u'profile.' % reverse( - profile, kwargs={'username': to_username})}} + "type": "alert-success", + "text": _( + f"Successfully cloned to {form_url_link} into your " + f"{profile_url_link}" + ), + } + return {} form_result = publish_form(set_form) - if form_result['type'] == 'alert-success': + if form_result["type"] == "alert-success": # comment the following condition (and else) # when we want to enable sms check for all. # until then, it checks if form barely related to sms - if is_sms_related(form_result.get('form_o')): + if is_sms_related(form_result.get("form_o")): form_result_sms = check_form_sms_compatibility(form_result) message_list = [form_result, form_result_sms] else: @@ -149,69 +181,72 @@ def set_form(): else: message = form_result - context = RequestContext(request, { - 'message': message, 'message_list': message_list}) + context = {"message": message, "message_list": message_list} if request.is_ajax(): - res = loader.render_to_string( - 'message.html', - context_instance=context - ).replace("'", r"\'").replace('\n', '') + res = ( + loader.render_to_string("message.html", context=context, request=request) + .replace("'", r"\'") + .replace("\n", "") + ) - return HttpResponse( - "$('#mfeedback').html('%s').show();" % res) - else: - return HttpResponse(message['text']) + return HttpResponse(f"$('#mfeedback').html('{res}').show();") + + return HttpResponse(message["text"]) +# pylint: disable=too-many-locals def profile(request, username): + """Show user profiles page view.""" content_user = get_object_or_404(User, username__iexact=username) form = QuickConverter() - data = {'form': form} + data = {"form": form} # xlsform submission... - if request.method == 'POST' and request.user.is_authenticated: + if request.method == "POST" and request.user.is_authenticated: + def set_form(): form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user).survey audit = {} audit_log( - Actions.FORM_PUBLISHED, request.user, content_user, - _("Published form '%(id_string)s'.") % - { - 'id_string': survey.id_string, - }, audit, request) + Actions.FORM_PUBLISHED, + request.user, + content_user, + _(f"Published form '{survey.id_string}'."), + audit, + request, + ) enketo_webform_url = reverse( - enter_data, - kwargs={'username': username, 'id_string': survey.id_string} + enter_data, kwargs={"username": username, "id_string": survey.id_string} ) return { - 'type': 'alert-success', - 'preview_url': reverse(enketo_preview, kwargs={ - 'username': username, - 'id_string': survey.id_string - }), - 'text': _(u'Successfully published %(form_id)s.' - u' Enter Web Form' - u' or ' - u'Preview Web Form') % { - 'form_id': survey.id_string, - 'form_url': enketo_webform_url - }, - 'form_o': survey + "type": "alert-success", + "preview_url": reverse( + enketo_preview, + kwargs={"username": username, "id_string": survey.id_string}, + ), + "text": _( + f"Successfully published {survey.id_string}." + f' Enter Web Form' + ' or ' + "Preview Web Form" + ), + "form_o": survey, } + form_result = publish_form(set_form) - if form_result['type'] == 'alert-success': + if form_result["type"] == "alert-success": # comment the following condition (and else) # when we want to enable sms check for all. # until then, it checks if form barely related to sms - if is_sms_related(form_result.get('form_o')): + if is_sms_related(form_result.get("form_o")): form_result_sms = check_form_sms_compatibility(form_result) - data['message_list'] = [form_result, form_result_sms] + data["message_list"] = [form_result, form_result_sms] else: - data['message'] = form_result + data["message"] = form_result else: - data['message'] = form_result + data["message"] = form_result # profile view... # for the same user -> dashboard @@ -221,47 +256,68 @@ def set_form(): form = QuickConverterFile() form_url = QuickConverterURL() - request_url = request.build_absolute_uri( - "/%s" % request.user.username) - url = request_url.replace('http://', 'https://') - xforms = XForm.objects.filter(user=content_user, - deleted_at__isnull=True)\ - .select_related('user').only( - 'id', 'id_string', 'downloadable', 'shared', 'shared_data', - 'user__username', 'num_of_submissions', 'title', - 'last_submission_time', 'instances_with_geopoints', - 'encrypted', 'date_created') + request_url = request.build_absolute_uri(f"/{request.user.username}") + url = request_url.replace("http://", "https://") + xforms = ( + XForm.objects.filter(user=content_user, deleted_at__isnull=True) + .select_related("user") + .only( + "id", + "id_string", + "downloadable", + "shared", + "shared_data", + "user__username", + "num_of_submissions", + "title", + "last_submission_time", + "instances_with_geopoints", + "encrypted", + "date_created", + ) + ) user_xforms = xforms # forms shared with user forms_shared_with = get_forms_shared_with_user(content_user).only( - 'id', 'id_string', 'downloadable', 'shared', 'shared_data', - 'user__username', 'num_of_submissions', 'title', - 'last_submission_time', 'instances_with_geopoints', 'encrypted', - 'date_created') + "id", + "id_string", + "downloadable", + "shared", + "shared_data", + "user__username", + "num_of_submissions", + "title", + "last_submission_time", + "instances_with_geopoints", + "encrypted", + "date_created", + ) xforms_list = [ { - 'id': 'published', - 'xforms': user_xforms, - 'title': _(u"Published Forms"), - 'small': _("Export, map, and view submissions.") + "id": "published", + "xforms": user_xforms, + "title": _("Published Forms"), + "small": _("Export, map, and view submissions."), }, { - 'id': 'shared', - 'xforms': forms_shared_with, - 'title': _(u"Shared Forms"), - 'small': _("List of forms shared with you.") - } + "id": "shared", + "xforms": forms_shared_with, + "title": _("Shared Forms"), + "small": _("List of forms shared with you."), + }, ] - data.update({ - 'all_forms': all_forms, - 'show_dashboard': show_dashboard, - 'form': form, - 'form_url': form_url, - 'url': url, - 'user_xforms': user_xforms, - 'xforms_list': xforms_list, - 'forms_shared_with': forms_shared_with - }) + data.update( + { + "all_forms": all_forms, + "show_dashboard": show_dashboard, + "form": form, + "form_url": form_url, + "url": url, + "user_xforms": user_xforms, + "xforms_list": xforms_list, + "forms_shared_with": forms_shared_with, + } + ) # for any other user -> profile set_profile_data(data, content_user) @@ -274,67 +330,81 @@ def set_form(): def members_list(request): + """Show members list page view.""" if not request.user.is_staff and not request.user.is_superuser: - return HttpResponseForbidden(_(u'Forbidden.')) + return HttpResponseForbidden(_("Forbidden.")) users = User.objects.all() - template = 'people.html' + template = "people.html" - return render(request, template, {'template': template, 'users': users}) + return render(request, template, {"template": template, "users": users}) @login_required def profile_settings(request, username): + """User profile settings page view.""" if request.user.username != username: return HttpResponseNotFound("Page not found") content_user = check_and_set_user(request, username) - profile, created = UserProfile.objects.get_or_create(user=content_user) - if request.method == 'POST': - form = UserProfileForm(request.POST, instance=profile) + user_profile, _created = UserProfile.objects.get_or_create(user=content_user) + if request.method == "POST": + form = UserProfileForm(request.POST, instance=user_profile) if form.is_valid(): # get user # user.email = cleaned_email - form.instance.user.email = form.cleaned_data['email'] + form.instance.user.email = form.cleaned_data["email"] form.instance.user.save() form.save() # todo: add string rep. of settings to see what changed audit = {} audit_log( - Actions.PROFILE_SETTINGS_UPDATED, request.user, content_user, - _("Profile settings updated."), audit, request) - return HttpResponseRedirect(reverse( - public_profile, kwargs={'username': request.user.username} - )) + Actions.PROFILE_SETTINGS_UPDATED, + request.user, + content_user, + _("Profile settings updated."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(public_profile, kwargs={"username": request.user.username}) + ) else: - form = UserProfileForm( - instance=profile, initial={"email": content_user.email}) + form = UserProfileForm(instance=profile, initial={"email": content_user.email}) - return render(request, "settings.html", - {'content_user': content_user, 'form': form}) + return render( + request, "settings.html", {"content_user": content_user, "form": form} + ) @require_GET def public_profile(request, username): + """Show user's public profile page view.""" content_user = check_and_set_user(request, username) if isinstance(content_user, HttpResponseRedirect): return content_user data = {} set_profile_data(data, content_user) - data['is_owner'] = request.user == content_user + data["is_owner"] = request.user == content_user audit = {} audit_log( - Actions.PUBLIC_PROFILE_ACCESSED, request.user, content_user, - _("Public profile accessed."), audit, request) + Actions.PUBLIC_PROFILE_ACCESSED, + request.user, + content_user, + _("Public profile accessed."), + audit, + request, + ) return render(request, "profile.html", data) @login_required def dashboard(request): + """Show the dashboard page view.""" content_user = request.user data = { - 'form': QuickConverter(), - 'content_user': content_user, - 'url': request.build_absolute_uri("/%s" % request.user.username) + "form": QuickConverter(), + "content_user": content_user, + "url": request.build_absolute_uri(f"/{request.user.username}"), } set_profile_data(data, content_user) @@ -342,95 +412,107 @@ def dashboard(request): def redirect_to_public_link(request, uuid): + """Redirects to the public link of the form.""" xform = get_object_or_404(XForm, uuid=uuid, deleted_at__isnull=True) - request.session['public_link'] = \ + request.session["public_link"] = ( xform.uuid if MetaData.public_link(xform) else False + ) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': xform.user.username, - 'id_string': xform.id_string - })) + return HttpResponseRedirect( + reverse( + show, kwargs={"username": xform.user.username, "id_string": xform.id_string} + ) + ) def set_xform_owner_data(data, xform, request, username, id_string): - data['sms_support_form'] = ActivateSMSSupportForm( - initial={'enable_sms_support': xform.allows_sms, - 'sms_id_string': xform.sms_id_string}) + """Set xform owner page view.""" + data["sms_support_form"] = ActivateSMSSupportForm( + initial={ + "enable_sms_support": xform.allows_sms, + "sms_id_string": xform.sms_id_string, + } + ) if not xform.allows_sms: - data['sms_compatible'] = check_form_sms_compatibility( - None, json_survey=json.loads(xform.json)) + data["sms_compatible"] = check_form_sms_compatibility( + None, json_survey=json.loads(xform.json) + ) else: - url_root = request.build_absolute_uri('/')[:-1] - data['sms_providers_doc'] = providers_doc( - url_root=url_root, - username=username, - id_string=id_string) - data['url_root'] = url_root - - data['form_license_form'] = FormLicenseForm( - initial={'value': data['form_license']}) - data['data_license_form'] = DataLicenseForm( - initial={'value': data['data_license']}) - data['doc_form'] = SupportDocForm() - data['source_form'] = SourceForm() - data['media_form'] = MediaForm() - data['mapbox_layer_form'] = MapboxLayerForm() - data['external_export_form'] = ExternalExportForm() + url_root = request.build_absolute_uri("/")[:-1] + data["sms_providers_doc"] = providers_doc( + url_root=url_root, username=username, id_string=id_string + ) + data["url_root"] = url_root + + data["form_license_form"] = FormLicenseForm(initial={"value": data["form_license"]}) + data["data_license_form"] = DataLicenseForm(initial={"value": data["data_license"]}) + data["doc_form"] = SupportDocForm() + data["source_form"] = SourceForm() + data["media_form"] = MediaForm() + data["mapbox_layer_form"] = MapboxLayerForm() + data["external_export_form"] = ExternalExportForm() users_with_perms = [] for perm in get_xform_users_with_perms(xform).items(): has_perm = [] - if 'change_xform' in perm[1]: - has_perm.append(_(u"Can Edit")) - if 'view_xform' in perm[1]: - has_perm.append(_(u"Can View")) - if 'report_xform' in perm[1]: - has_perm.append(_(u"Can submit to")) - users_with_perms.append((perm[0], u" | ".join(has_perm))) - data['users_with_perms'] = users_with_perms - data['permission_form'] = PermissionForm(username) + if "change_xform" in perm[1]: + has_perm.append(_("Can Edit")) + if "view_xform" in perm[1]: + has_perm.append(_("Can View")) + if "report_xform" in perm[1]: + has_perm.append(_("Can submit to")) + users_with_perms.append((perm[0], " | ".join(has_perm))) + data["users_with_perms"] = users_with_perms + data["permission_form"] = PermissionForm(username) @require_GET def show(request, username=None, id_string=None, uuid=None): + """Show form page view.""" if uuid: return redirect_to_public_link(request, uuid) - xform, is_owner, can_edit, can_view = get_xform_and_perms( - username, id_string, request) + xform, is_xform_owner, can_edit, can_view = get_xform_and_perms( + username, id_string, request + ) # no access - if not (xform.shared or can_view or request.session.get('public_link')): + if not (xform.shared or can_view or request.session.get("public_link")): return HttpResponseRedirect(reverse(home)) data = {} - data['cloned'] = len( - XForm.objects.filter(user__username__iexact=request.user.username, - id_string__iexact=id_string + XForm.CLONED_SUFFIX, - deleted_at__isnull=True) - ) > 0 + data["cloned"] = ( + len( + XForm.objects.filter( + user__username__iexact=request.user.username, + id_string__iexact=id_string + XForm.CLONED_SUFFIX, + deleted_at__isnull=True, + ) + ) + > 0 + ) try: - data['public_link'] = MetaData.public_link(xform) - data['is_owner'] = is_owner - data['can_edit'] = can_edit - data['can_view'] = can_view or request.session.get('public_link') - data['xform'] = xform - data['content_user'] = xform.user - data['base_url'] = "https://%s" % request.get_host() - data['source'] = MetaData.source(xform) - data['form_license'] = MetaData.form_license(xform) - data['data_license'] = MetaData.data_license(xform) - data['supporting_docs'] = MetaData.supporting_docs(xform) - data['media_upload'] = MetaData.media_upload(xform) - data['mapbox_layer'] = MetaData.mapbox_layer_upload(xform) - data['external_export'] = MetaData.external_export(xform) + data["public_link"] = MetaData.public_link(xform) + data["is_owner"] = is_xform_owner + data["can_edit"] = can_edit + data["can_view"] = can_view or request.session.get("public_link") + data["xform"] = xform + data["content_user"] = xform.user + data["base_url"] = f"https://{request.get_host()}" + data["source"] = MetaData.source(xform) + data["form_license"] = MetaData.form_license(xform) + data["data_license"] = MetaData.data_license(xform) + data["supporting_docs"] = MetaData.supporting_docs(xform) + data["media_upload"] = MetaData.media_upload(xform) + data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) + data["external_export"] = MetaData.external_export(xform) except XLSFormError as e: return HttpResponseBadRequest(e.__str__()) - if is_owner: + if is_xform_owner: set_xform_owner_data(data, xform, request, username, id_string) if xform.allows_sms: - data['sms_support_doc'] = get_autodoc_for(xform) + data["sms_support_doc"] = get_autodoc_for(xform) return render(request, "show.html", data) @@ -438,16 +520,18 @@ def show(request, username=None, id_string=None, uuid=None): @login_required @require_GET def api_token(request, username=None): + """Show user's API Token page view.""" if request.user.username == username: user = get_object_or_404(User, username=username) data = {} - data['token_key'], created = Token.objects.get_or_create(user=user) + data["token_key"], _created = Token.objects.get_or_create(user=user) return render(request, "api_token.html", data) - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) +# pylint: disable=too-many-locals,too-many-branches @require_http_methods(["GET", "OPTIONS"]) def api(request, username=None, id_string=None): """ @@ -471,25 +555,25 @@ def api(request, username=None, id_string=None): helper_auth_helper(request) helper_auth_helper(request) - xform, owner = check_and_set_user_and_form(username, id_string, request) + xform, _owner = check_and_set_user_and_form(username, id_string, request) if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - query = request.GET.get('query') + query = request.GET.get("query") total_records = xform.num_of_submissions try: args = { - 'xform': xform, - 'query': query, - 'fields': request.GET.get('fields'), - 'sort': request.GET.get('sort') + "xform": xform, + "query": query, + "fields": request.GET.get("fields"), + "sort": request.GET.get("sort"), } - if 'page' in request.GET: - page = int(request.GET.get('page')) - page_size = request.GET.get('page_size', request.GET.get('limit')) + if "page" in request.GET: + page = int(request.GET.get("page")) + page_size = request.GET.get("page_size", request.GET.get("limit")) if page_size: page_size = int(page_size) @@ -503,36 +587,32 @@ def api(request, username=None, id_string=None): if query: count_args = args.copy() - count_args['count'] = True - count_results = [i for i in query_data(**count_args)] - - if len(count_results): - total_records = count_results[0].get('count', total_records) + count_args["count"] = True + count_results = list(query_data(**count_args)) + if count_results: + total_records = count_results[0].get("count", total_records) - if 'start' in request.GET: - args["start_index"] = int(request.GET.get('start')) + if "start" in request.GET: + args["start_index"] = int(request.GET.get("start")) - if 'limit' in request.GET: - args["limit"] = int(request.GET.get('limit')) + if "limit" in request.GET: + args["limit"] = int(request.GET.get("limit")) - if 'count' in request.GET: - args["count"] = True if int(request.GET.get('count')) > 0\ - else False + if "count" in request.GET: + args["count"] = int(request.GET.get("count")) > 0 cursor = query_data(**args) except (ValueError, TypeError) as e: return HttpResponseBadRequest(conditional_escape(e.__str__())) - try: - response_text = json_util.dumps([i for i in cursor]) - except OperationalError: - return HttpResponseServerError() - - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") + response_text = json_util.dumps(list(cursor)) + response_text = f"{callback}({response_text})" + response = HttpResponse(response_text) + else: + response = JsonResponse(list(cursor)) - response = HttpResponse(response_text, content_type='application/json') add_cors_headers(response) return response @@ -543,343 +623,355 @@ def public_api(request, username, id_string): """ Returns public information about the form as JSON """ - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) - _DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' + datetime_format = "%Y-%m-%d %H:%M:%S" exports = { - 'username': xform.user.username, - 'id_string': xform.id_string, - 'bamboo_dataset': xform.bamboo_dataset, - 'shared': xform.shared, - 'shared_data': xform.shared_data, - 'downloadable': xform.downloadable, - 'title': xform.title, - 'date_created': xform.date_created.strftime(_DATETIME_FORMAT), - 'date_modified': xform.date_modified.strftime(_DATETIME_FORMAT), - 'uuid': xform.uuid, + "username": xform.user.username, + "id_string": xform.id_string, + "bamboo_dataset": xform.bamboo_dataset, + "shared": xform.shared, + "shared_data": xform.shared_data, + "downloadable": xform.downloadable, + "title": xform.title, + "date_created": xform.date_created.strftime(datetime_format), + "date_modified": xform.date_modified.strftime(datetime_format), + "uuid": xform.uuid, } - response_text = json.dumps(exports) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse(exports) +# pylint: disable=too-many-locals,too-many-branches,too-many-statements @login_required def edit(request, username, id_string): - xform = XForm.objects.get(user__username__iexact=username, - id_string__iexact=id_string, - deleted_at__isnull=True) + """Edit form page view.""" + xform = XForm.objects.get( + user__username__iexact=username, + id_string__iexact=id_string, + deleted_at__isnull=True, + ) owner = xform.user - if username == request.user.username or\ - request.user.has_perm('logger.change_xform', xform): - if request.POST.get('description') or\ - request.POST.get('description') == '': - audit = { - 'xform': xform.id_string - } + if username == request.user.username or request.user.has_perm( + "logger.change_xform", xform + ): + if request.POST.get("description") or request.POST.get("description") == "": + audit = {"xform": xform.id_string} + old_description = xform.description + new_description = request.POST["description"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Description for '%(id_string)s' updated from " - "'%(old_description)s' to '%(new_description)s'.") % - { - 'id_string': xform.id_string, - 'old_description': xform.description, - 'new_description': request.POST['description'] - }, audit, request) - xform.description = request.POST['description'] - elif request.POST.get('title'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Description for '{id_string}' updated from " + f"'{old_description}' to '{new_description}'." + ), + audit, + request, + ) + xform.description = request.POST["description"] + elif request.POST.get("title"): + audit = {"xform": xform.id_string} + old_title = (xform.title,) + new_title = (request.POST.get("title"),) audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Title for '%(id_string)s' updated from " - "'%(old_title)s' to '%(new_title)s'.") % - { - 'id_string': xform.id_string, - 'old_title': xform.title, - 'new_title': request.POST.get('title') - }, audit, request) - xform.title = request.POST['title'] - elif request.POST.get('toggle_shared'): - if request.POST['toggle_shared'] == 'data': - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Title for '{id_string}' updated from " + f"'{old_title}' to '{new_title}'." + ) + % { + "id_string": xform.id_string, + }, + audit, + request, + ) + xform.title = request.POST["title"] + elif request.POST.get("toggle_shared"): + if request.POST["toggle_shared"] == "data": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.shared_data else _("not shared") + new_shared = _("shared") if not xform.shared_data else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Data sharing updated for '%(id_string)s' from " - "'%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.shared_data else _("not shared"), - 'new_shared': - _("shared") - if not xform.shared_data else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Data sharing updated for '{id_string}' from " + f"'{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.shared_data = not xform.shared_data - elif request.POST['toggle_shared'] == 'form': - audit = { - 'xform': xform.id_string - } + elif request.POST["toggle_shared"] == "form": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.shared else _("not shared") + new_shared = _("shared") if not xform.shared else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Form sharing for '%(id_string)s' updated " - "from '%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.shared else _("not shared"), - 'new_shared': - _("shared") if not xform.shared else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Form sharing for '{xform.id_string}' updated " + f"from '{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.shared = not xform.shared - elif request.POST['toggle_shared'] == 'active': - audit = { - 'xform': xform.id_string - } + elif request.POST["toggle_shared"] == "active": + audit = {"xform": xform.id_string} + old_shared = _("shared") if xform.downloadable else _("not shared") + new_shared = _("shared") if not xform.downloadable else _("not shared") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Active status for '%(id_string)s' updated from " - "'%(old_shared)s' to '%(new_shared)s'.") % - { - 'id_string': xform.id_string, - 'old_shared': - _("shared") if xform.downloadable else _("not shared"), - 'new_shared': - _("shared") - if not xform.downloadable else _("not shared") - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Active status for '{xform.id_string}' updated from " + f"'{old_shared}' to '{new_shared}'." + ), + audit, + request, + ) xform.downloadable = not xform.downloadable - elif request.POST.get('form-license'): - audit = { - 'xform': xform.id_string - } + elif request.POST.get("form-license"): + audit = {"xform": xform.id_string} + form_license = request.POST["form-license"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Form License for '%(id_string)s' updated to " - "'%(form_license)s'.") % - { - 'id_string': xform.id_string, - 'form_license': request.POST['form-license'], - }, audit, request) - MetaData.form_license(xform, request.POST['form-license']) - elif request.POST.get('data-license'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Form License for '{xform.id_string}' updated to " + f"'{form_license}'." + ), + audit, + request, + ) + MetaData.form_license(xform, request.POST["form-license"]) + elif request.POST.get("data-license"): + audit = {"xform": xform.id_string} + data_license = request.POST["data-license"] audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Data license for '%(id_string)s' updated to " - "'%(data_license)s'.") % - { - 'id_string': xform.id_string, - 'data_license': request.POST['data-license'], - }, audit, request) - MetaData.data_license(xform, request.POST['data-license']) - elif request.POST.get('source') or request.FILES.get('source'): - audit = { - 'xform': xform.id_string - } + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Data license for '{xform.id_string}' updated to " + f"'{data_license}'." + ), + audit, + request, + ) + MetaData.data_license(xform, request.POST["data-license"]) + elif request.POST.get("source") or request.FILES.get("source"): + audit = {"xform": xform.id_string} + source = request.POST.get("source") audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Source for '%(id_string)s' updated to '%(source)s'.") % - { - 'id_string': xform.id_string, - 'source': request.POST.get('source'), - }, audit, request) - MetaData.source(xform, request.POST.get('source'), - request.FILES.get('source')) - elif request.POST.get('enable_sms_support_trigger') is not None: + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Source for '{xform.id_string}' updated to '{source}'."), + audit, + request, + ) + MetaData.source( + xform, request.POST.get("source"), request.FILES.get("source") + ) + elif request.POST.get("enable_sms_support_trigger") is not None: sms_support_form = ActivateSMSSupportForm(request.POST) if sms_support_form.is_valid(): - audit = { - 'xform': xform.id_string - } - enabled = \ - sms_support_form.cleaned_data.get('enable_sms_support') + audit = {"xform": xform.id_string} + enabled = sms_support_form.cleaned_data.get("enable_sms_support") if enabled: audit_action = Actions.SMS_SUPPORT_ACTIVATED - audit_message = _(u"SMS Support Activated on") + audit_message = _("SMS Support Activated on") else: audit_action = Actions.SMS_SUPPORT_DEACTIVATED - audit_message = _(u"SMS Support Deactivated on") + audit_message = _("SMS Support Deactivated on") audit_log( - audit_action, request.user, owner, - audit_message - % {'id_string': xform.id_string}, audit, request) + audit_action, + request.user, + owner, + audit_message, + audit, + request, + ) # stored previous states to be able to rollback form status # in case we can't save. - pe = xform.allows_sms + previous_allow_sms = xform.allows_sms pid = xform.sms_id_string xform.allows_sms = enabled - xform.sms_id_string = \ - sms_support_form.cleaned_data.get('sms_id_string') - compat = check_form_sms_compatibility(None, - json.loads(xform.json)) - if compat['type'] == 'alert-error': + xform.sms_id_string = sms_support_form.cleaned_data.get("sms_id_string") + compat = check_form_sms_compatibility(None, json.loads(xform.json)) + if compat["type"] == "alert-error": xform.allows_sms = False xform.sms_id_string = pid try: xform.save() except IntegrityError: # unfortunately, there's no feedback mechanism here - xform.allows_sms = pe + xform.allows_sms = previous_allow_sms xform.sms_id_string = pid - elif request.POST.get('media_url'): - uri = request.POST.get('media_url') + elif request.POST.get("media_url"): + uri = request.POST.get("media_url") MetaData.media_add_uri(xform, uri) - elif request.FILES.get('media'): - audit = { - 'xform': xform.id_string - } + elif request.FILES.get("media"): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - for aFile in request.FILES.getlist("media"): - MetaData.media_upload(xform, aFile) - elif request.POST.get('map_name'): + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Media added to '{xform.id_string}'."), + audit, + request, + ) + for media_file in request.FILES.getlist("media"): + MetaData.media_upload(xform, media_file) + elif request.POST.get("map_name"): mapbox_layer = MapboxLayerForm(request.POST) if mapbox_layer.is_valid(): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Map layer added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Map layer added to '{xform.id_string}'."), + audit, + request, + ) MetaData.mapbox_layer_upload(xform, mapbox_layer.cleaned_data) - elif request.FILES.get('doc'): - audit = { - 'xform': xform.id_string - } + elif request.FILES.get("doc"): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Supporting document added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - MetaData.supporting_docs(xform, request.FILES.get('doc')) - elif request.POST.get("template_token") \ - and request.POST.get("template_token"): + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Supporting document added to '{xform.id_string}'."), + audit, + request, + ) + MetaData.supporting_docs(xform, request.FILES.get("doc")) + elif request.POST.get("template_token") and request.POST.get("template_token"): template_name = request.POST.get("template_name") template_token = request.POST.get("template_token") - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("External export added to '%(id_string)s'.") % - { - 'id_string': xform.id_string - }, audit, request) - merged = template_name + '|' + template_token + Actions.FORM_UPDATED, + request.user, + owner, + _(f"External export added to '{xform.id_string}'."), + audit, + request, + ) + merged = template_name + "|" + template_token MetaData.external_export(xform, merged) - elif request.POST.get("external_url") \ - and request.FILES.get("xls_template"): + elif request.POST.get("external_url") and request.FILES.get("xls_template"): template_upload_name = request.POST.get("template_upload_name") external_url = request.POST.get("external_url") xls_template = request.FILES.get("xls_template") - result = upload_template_for_external_export(external_url, - xls_template) - status_code = result.split('|')[0] - token = result.split('|')[1] - if status_code == '201': - data_value =\ - template_upload_name + '|' + external_url + '/xls/' + token + result = upload_template_for_external_export(external_url, xls_template) + status_code = result.split("|")[0] + token = result.split("|")[1] + if status_code == "201": + data_value = template_upload_name + "|" + external_url + "/xls/" + token MetaData.external_export(xform, data_value=data_value) xform.update() if request.is_ajax(): - return HttpResponse(_(u'Updated succeeded.')) - else: - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponse(_("Updated succeeded.")) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) - return HttpResponseForbidden(_(u'Update failed.')) + return HttpResponseForbidden(_("Update failed.")) def getting_started(request): - template = 'getting_started.html' + """The getting started page view.""" + template = "getting_started.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def support(request): - template = 'support.html' + """The support page view.""" + template = "support.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def faq(request): - template = 'faq.html' + """The frequently asked questions page view.""" + template = "faq.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def xls2xform(request): - template = 'xls2xform.html' + """The XLSForm to XForm page view.""" + template = "xls2xform.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def tutorial(request): - template = 'tutorial.html' - username = request.user.username if request.user.username else \ - 'your-user-name' - url = request.build_absolute_uri("/%s" % username) + """The tutorial page view.""" + template = "tutorial.html" + username = request.user.username if request.user.username else "your-user-name" + url = request.build_absolute_uri(f"/{username}") - return render(request, 'base.html', {'template': template, 'url': url}) + return render(request, "base.html", {"template": template, "url": url}) def resources(request): - if 'fr' in request.LANGUAGE_CODE.lower(): - deck_id = 'a351f6b0a3730130c98b12e3c5740641' - else: - deck_id = '1a33a070416b01307b8022000a1de118' + """The resources page view.""" + deck_id = "1a33a070416b01307b8022000a1de118" + if "fr" in request.LANGUAGE_CODE.lower(): + deck_id = "a351f6b0a3730130c98b12e3c5740641" - return render(request, 'resources.html', {'deck_id': deck_id}) + return render(request, "resources.html", {"deck_id": deck_id}) def about_us(request): - a_flatpage = '/about-us/' - username = request.user.username if request.user.username else \ - 'your-user-name' - url = request.build_absolute_uri("/%s" % username) + """The about us page view""" + a_flatpage = "/about-us/" + username = request.user.username if request.user.username else "your-user-name" + url = request.build_absolute_uri(f"/{username}") - return render(request, 'base.html', {'a_flatpage': a_flatpage, 'url': url}) + return render(request, "base.html", {"a_flatpage": a_flatpage, "url": url}) def privacy(request): - template = 'privacy.html' + """The privacy page view.""" + template = "privacy.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def tos(request): - template = 'tos.html' + """The terms of service page view.""" + template = "tos.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def syntax(request): - template = 'syntax.html' + """The XLSForm Syntax page view.""" + template = "syntax.html" - return render(request, 'base.html', {'template': template}) + return render(request, "base.html", {"template": template}) def form_gallery(request): @@ -889,363 +981,372 @@ def form_gallery(request): """ data = {} if request.user.is_authenticated: - data['loggedin_user'] = request.user - data['shared_forms'] = XForm.objects.filter(shared=True, - deleted_at__isnull=True) + data["loggedin_user"] = request.user + data["shared_forms"] = XForm.objects.filter(shared=True, deleted_at__isnull=True) # build list of shared forms with cloned suffix id_strings_with_cloned_suffix = [ - x.id_string + XForm.CLONED_SUFFIX for x in data['shared_forms'] + x.id_string + XForm.CLONED_SUFFIX for x in data["shared_forms"] ] # build list of id_strings for forms this user has cloned - data['cloned'] = [ + data["cloned"] = [ x.id_string.split(XForm.CLONED_SUFFIX)[0] for x in XForm.objects.filter( user__username__iexact=request.user.username, id_string__in=id_strings_with_cloned_suffix, - deleted_at__isnull=True + deleted_at__isnull=True, ) ] - return render(request, 'form_gallery.html', data) + return render(request, "form_gallery.html", data) def download_metadata(request, username, id_string, data_id): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Downloads metadata file contents.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user if username == request.user.username or xform.shared: data = get_object_or_404(MetaData, pk=data_id) file_path = data.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") dfs = get_storage_class()() if dfs.exists(file_path): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + filename_w_extension = f"{filename}.{extension}" audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Document '%(filename)s' for '%(id_string)s' downloaded.") % - { - 'id_string': xform.id_string, - 'filename': "%s.%s" % (filename, extension) - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Document '{filename_w_extension}' for " + f"'{xform.id_string}' downloaded." + ), + audit, + request, + ) response = response_with_mimetype_and_name( data.data_file_type, - filename, extension=extension, show_date=False, - file_path=file_path) + filename, + extension=extension, + show_date=False, + file_path=file_path, + ) return response - else: - return HttpResponseNotFound() + return HttpResponseNotFound() - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) @login_required() def delete_metadata(request, username, id_string, data_id): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Deletes a metadata record.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user data = get_object_or_404(MetaData, pk=data_id) dfs = get_storage_class()() req_username = request.user.username - if request.GET.get('del', False) and username == req_username: - try: - dfs.delete(data.data_file.name) - data.delete() - audit = { - 'xform': xform.id_string - } - audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Document '%(filename)s' deleted from '%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'filename': os.path.basename(data.data_file.name) - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) - except Exception: - return HttpResponseServerError() - elif (request.GET.get('map_name_del', False) or - request.GET.get('external_del', False)) and username == req_username: + if request.GET.get("del", False) and username == req_username: + dfs.delete(data.data_file.name) data.delete() - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + filename = (os.path.basename(data.data_file.name),) audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Map layer deleted from '%(id_string)s'.") % - { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Document '{filename}' deleted from '{xform.id_string}'."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) + if ( + request.GET.get("map_name_del", False) or request.GET.get("external_del", False) + ) and username == req_username: + data.delete() + audit = {"xform": xform.id_string} + audit_log( + Actions.FORM_UPDATED, + request.user, + owner, + _(f"Map layer deleted from '{xform.id_string}'."), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) def download_media_data(request, username, id_string, data_id): + """Redirects to a form metadata record for download.""" xform = get_object_or_404( - XForm, user__username__iexact=username, deleted_at__isnull=True, - id_string__iexact=id_string) + XForm, + user__username__iexact=username, + deleted_at__isnull=True, + id_string__iexact=id_string, + ) owner = xform.user data = get_object_or_404(MetaData, id=data_id) dfs = get_storage_class()() - if request.GET.get('del', False): + if request.GET.get("del", False): if username == request.user.username: - try: - # ensure filename is not an empty string - if data.data_file.name != '': - dfs.delete(data.data_file.name) - - data.delete() - audit = { - 'xform': xform.id_string - } - audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media download '%(filename)s' deleted from " - "'%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'filename': os.path.basename(data.data_file.name) - }, audit, request) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) - except Exception as e: - return HttpResponseServerError(e) + # ensure filename is not an empty string + if data.data_file.name != "": + dfs.delete(data.data_file.name) + + data.delete() + audit = {"xform": xform.id_string} + audit_log( + Actions.FORM_UPDATED, + request.user, + owner, + _( + f"Media download '{os.path.basename(data.data_file.name)}'" + f" deleted from '{xform.id_string}'." + ), + audit, + request, + ) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) else: if username: # == request.user.username or xform.shared: - if data.data_file.name == '' and data.data_value is not None: + if data.data_file.name == "" and data.data_value is not None: return HttpResponseRedirect(data.data_value) file_path = data.data_file.name - filename, extension = os.path.splitext(file_path.split('/')[-1]) - extension = extension.strip('.') + filename, extension = os.path.splitext(file_path.split("/")[-1]) + extension = extension.strip(".") if dfs.exists(file_path): - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_UPDATED, request.user, owner, - _("Media '%(filename)s' downloaded from '%(id_string)s'.") - % { - 'id_string': xform.id_string, - 'filename': os.path.basename(file_path) - }, audit, request) + Actions.FORM_UPDATED, + request.user, + owner, + _( + "Media f'{os.path.basename(file_path)}' " + f"downloaded from '{xform.id_string}'." + ), + audit, + request, + ) response = response_with_mimetype_and_name( data.data_file_type, - filename, extension=extension, show_date=False, - file_path=file_path) + filename, + extension=extension, + show_date=False, + file_path=file_path, + ) return response - else: - return HttpResponseNotFound() + return HttpResponseNotFound() - return HttpResponseForbidden(_(u'Permission denied.')) + return HttpResponseForbidden(_("Permission denied.")) def form_photos(request, username, id_string): + """View form image attachments.""" xform, owner = check_and_set_user_and_form(username, id_string, request) if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) data = {} - data['form_view'] = True - data['content_user'] = owner - data['xform'] = xform + data["form_view"] = True + data["content_user"] = owner + data["xform"] = xform image_urls = [] for instance in xform.instances.filter(deleted_at__isnull=True): for attachment in instance.attachments.all(): # skip if not image e.g video or file - if not attachment.mimetype.startswith('image'): + if not attachment.mimetype.startswith("image"): continue data = {} - for i in [u'small', u'medium', u'large', u'original']: - url = reverse(attachment_url, kwargs={'size': i}) - url = '%s?media_file=%s' % (url, attachment.media_file.name) + for i in ["small", "medium", "large", "original"]: + url = reverse(attachment_url, kwargs={"size": i}) + url = f"{url}?media_file={attachment.media_file.name}" data[i] = url image_urls.append(data) image_urls = json.dumps(image_urls) - data['images'] = image_urls - data['profile'], created = UserProfile.objects.get_or_create(user=owner) + data["images"] = image_urls + data["profile"], _created = UserProfile.objects.get_or_create(user=owner) - return render(request, 'form_photos.html', data) + return render(request, "form_photos.html", data) +# pylint: disable=too-many-branches @require_POST def set_perm(request, username, id_string): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Assign form permissions to a user.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user - if username != request.user.username\ - and not has_permission(xform, username, request): - return HttpResponseForbidden(_(u'Permission denied.')) + if username != request.user.username and not has_permission( + xform, username, request + ): + return HttpResponseForbidden(_("Permission denied.")) try: - perm_type = request.POST['perm_type'] - for_user = request.POST['for_user'] + perm_type = request.POST["perm_type"] + for_user = request.POST["for_user"] except KeyError: return HttpResponseBadRequest() - if perm_type in ['edit', 'view', 'report', 'remove']: + if perm_type in ["edit", "view", "report", "remove"]: try: user = User.objects.get(username=for_user) except User.DoesNotExist: messages.add_message( - request, messages.INFO, - _(u"Wrong username %s." % for_user), - extra_tags='alert-error') + request, + messages.INFO, + _(f"Wrong username {for_user}."), + extra_tags="alert-error", + ) else: - if perm_type == 'edit' and\ - not user.has_perm('change_xform', xform): - audit = { - 'xform': xform.id_string - } + if perm_type == "edit" and not user.has_perm("change_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Edit permissions on '%(id_string)s' assigned to " - "'%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('change_xform', user, xform) - elif perm_type == 'view' and\ - not user.has_perm('view_xform', xform): - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"Edit permissions on '{xform.id_string}' assigned to " + f"'{for_user}'." + ), + audit, + request, + ) + assign_perm("change_xform", user, xform) + elif perm_type == "view" and not user.has_perm("view_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("View permissions on '%(id_string)s' " - "assigned to '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('view_xform', user, xform) - elif perm_type == 'report' and\ - not user.has_perm('report_xform', xform): - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"View permissions on '{xform.id_string}' " + f"assigned to '{for_user}'." + ) + % {"id_string": xform.id_string, "for_user": for_user}, + audit, + request, + ) + assign_perm("view_xform", user, xform) + elif perm_type == "report" and not user.has_perm("report_xform", xform): + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Report permissions on '%(id_string)s' " - "assigned to '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - assign_perm('report_xform', user, xform) - elif perm_type == 'remove': - audit = { - 'xform': xform.id_string - } + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"Report permissions on '{xform.id_string}' " + f"assigned to '{for_user}'." + ), + audit, + request, + ) + assign_perm("report_xform", user, xform) + elif perm_type == "remove": + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("All permissions on '%(id_string)s' " - "removed from '%(for_user)s'.") % - { - 'id_string': xform.id_string, - 'for_user': for_user - }, audit, request) - remove_perm('change_xform', user, xform) - remove_perm('view_xform', user, xform) - remove_perm('report_xform', user, xform) - elif perm_type == 'link': + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _( + f"All permissions on '{xform.id_string}' " + f"removed from '{for_user}'." + ), + audit, + request, + ) + remove_perm("change_xform", user, xform) + remove_perm("view_xform", user, xform) + remove_perm("report_xform", user, xform) + elif perm_type == "link": current = MetaData.public_link(xform) - if for_user == 'all': + if for_user == "all": MetaData.public_link(xform, True) - elif for_user == 'none': + elif for_user == "none": MetaData.public_link(xform, False) - elif for_user == 'toggle': + elif for_user == "toggle": MetaData.public_link(xform, not current) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} + action = "removed" + if for_user == "all" or (for_user == "toggle" and not current): + action = "created" audit_log( - Actions.FORM_PERMISSIONS_UPDATED, request.user, owner, - _("Public link on '%(id_string)s' %(action)s.") % - { - 'id_string': xform.id_string, - 'action': - "created" if for_user == "all" or - (for_user == "toggle" and not current) else "removed" - }, audit, request) + Actions.FORM_PERMISSIONS_UPDATED, + request.user, + owner, + _(f"Public link on '{xform.id_string}' {action}."), + audit, + request, + ) if request.is_ajax(): - return HttpResponse( - json.dumps( - {'status': 'success'}), content_type='application/json') + return JsonResponse({"status": "success"}) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) @require_POST @login_required def delete_data(request, username=None, id_string=None): + """Delete submission record.""" xform, owner = check_and_set_user_and_form(username, id_string, request) - response_text = u'' + response_text = "" if not xform: - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) - data_id = request.POST.get('id') + data_id = request.POST.get("id") if not data_id: - return HttpResponseBadRequest(_(u"id must be specified")) + return HttpResponseBadRequest(_("id must be specified")) Instance.set_deleted_at(data_id, user=request.user) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.SUBMISSION_DELETED, request.user, owner, - _("Deleted submission with id '%(record_id)s' on '%(id_string)s'.") % - { - 'id_string': xform.id_string, - 'record_id': data_id - }, audit, request) - response_text = json.dumps({"success": "Deleted data %s" % data_id}) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) - - return HttpResponse(response_text, content_type='application/json') + Actions.SUBMISSION_DELETED, + request.user, + owner, + _(f"Deleted submission with id '{data_id}' on '{xform.id_string}'."), + audit, + request, + ) + response_data = {"success": f"Deleted data {data_id}"} + if "callback" in request.GET and request.GET.get("callback") != "": + response_text = json.dumps(response_data) + callback = request.GET.get("callback") + response_text = f"{callback}({response_text})" + return HttpResponse(response_text) + + return JsonResponse(response_data) @require_POST @is_owner def update_xform(request, username, id_string): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Update a form page view.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) owner = xform.user @@ -1254,170 +1355,153 @@ def set_form(): form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user, id_string).survey enketo_webform_url = reverse( - enter_data, - kwargs={'username': username, 'id_string': survey.id_string} + enter_data, kwargs={"username": username, "id_string": survey.id_string} ) - audit = { - 'xform': xform.id_string - } + audit = {"xform": xform.id_string} audit_log( - Actions.FORM_XLS_UPDATED, request.user, owner, - _("XLS for '%(id_string)s' updated.") % - { - 'id_string': xform.id_string, - }, audit, request) + Actions.FORM_XLS_UPDATED, + request.user, + owner, + _(f"XLS for '{xform.id_string}' updated."), + audit, + request, + ) return { - 'type': 'alert-success', - 'text': _(u'Successfully published %(form_id)s.' - u' Enter Web Form' - u' or ' - u'Preview Web Form') - % {'form_id': survey.id_string, - 'form_url': enketo_webform_url} + "type": "alert-success", + "text": _( + f"Successfully published {survey.id_string}." + f' Enter Web Form' + ' or ' + "Preview Web Form" + ), } + message = publish_form(set_form) messages.add_message( - request, messages.INFO, message['text'], extra_tags=message['type']) + request, messages.INFO, message["text"], extra_tags=message["type"] + ) - return HttpResponseRedirect(reverse(show, kwargs={ - 'username': username, - 'id_string': id_string - })) + return HttpResponseRedirect( + reverse(show, kwargs={"username": username, "id_string": id_string}) + ) @is_owner def activity(request, username): + """The activity/audit view for the given ``username``.""" owner = get_object_or_404(User, username=username) - return render(request, 'activity.html', {'user': owner}) + return render(request, "activity.html", {"user": owner}) def activity_fields(request): + """Returns Activity/Audit fields in JSON format.""" fields = [ { - 'id': 'created_on', - 'label': _('Performed On'), - 'type': 'datetime', - 'searchable': False - }, - { - 'id': 'action', - 'label': _('Action'), - 'type': 'string', - 'searchable': True, - 'options': sorted([Actions[e] for e in Actions.enums]) + "id": "created_on", + "label": _("Performed On"), + "type": "datetime", + "searchable": False, }, { - 'id': 'user', - 'label': 'Performed By', - 'type': 'string', - 'searchable': True - }, - { - 'id': 'msg', - 'label': 'Description', - 'type': 'string', - 'searchable': True + "id": "action", + "label": _("Action"), + "type": "string", + "searchable": True, + "options": sorted([Actions[e] for e in Actions.enums]), }, + {"id": "user", "label": "Performed By", "type": "string", "searchable": True}, + {"id": "msg", "label": "Description", "type": "string", "searchable": True}, ] - response_text = json.dumps(fields) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse(fields) @is_owner def activity_api(request, username): - from bson.objectid import ObjectId + """Returns Audit activity data in JSON format""" def stringify_unknowns(obj): + """Stringify some objects - for use with json.dumps.""" if isinstance(obj, ObjectId): return str(obj) if isinstance(obj, datetime): return obj.strftime(DATETIME_FORMAT) return None + try: - fields = request.GET.get('fields') - query = request.GET.get('query') - sort = request.GET.get('sort') + fields = request.GET.get("fields") + query = request.GET.get("query") + sort = request.GET.get("sort") query_args = { - 'username': username, - 'query': json.loads(query) if query else {}, - 'fields': json.loads(fields) if fields else [], - 'sort': json.loads(sort) if sort else [], + "username": username, + "query": json.loads(query) if query else {}, + "fields": json.loads(fields) if fields else [], + "sort": json.loads(sort) if sort else [], } - if 'start' in request.GET: - query_args["start"] = int(request.GET.get('start')) - if 'limit' in request.GET: - query_args["limit"] = int(request.GET.get('limit')) - if 'count' in request.GET: - query_args["count"] = True \ - if int(request.GET.get('count')) > 0 else False + if "start" in request.GET: + query_args["start"] = int(request.GET.get("start")) + if "limit" in request.GET: + query_args["limit"] = int(request.GET.get("limit")) + if "count" in request.GET: + query_args["count"] = int(request.GET.get("count")) > 0 cursor = AuditLog.query_data(**query_args) except ValueError as e: return HttpResponseBadRequest(e.__str__()) records = list(record for record in cursor) - response_text = json.dumps(records, default=stringify_unknowns) - if 'callback' in request.GET and request.GET.get('callback') != '': - callback = request.GET.get('callback') - response_text = ("%s(%s)" % (callback, response_text)) + if "callback" in request.GET and request.GET.get("callback") != "": + callback = request.GET.get("callback") + response_text = json.dumps(records, default=stringify_unknowns) + response_text = f"{callback}({response_text})" + return HttpResponse(response_text) - return HttpResponse(response_text, content_type='application/json') + return JsonResponse(records, json_dumps_params={"default": stringify_unknowns}) def qrcode(request, username, id_string): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Returns the Enketo URL in QR code image format.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) try: - formhub_url = "http://%s/" % request.META['HTTP_HOST'] + formhub_url = f"http://{request.META['HTTP_HOST']}/" except KeyError: formhub_url = "http://formhub.org/" - formhub_url = formhub_url + username + '/%s' % xform.pk + formhub_url = formhub_url + username + f"/{xform.pk}" if settings.TESTING_MODE: - formhub_url = "https://{}/{}".format(settings.TEST_HTTP_HOST, - settings.TEST_USERNAME) + formhub_url = f"https://{settings.TEST_HTTP_HOST}/{settings.TEST_USERNAME}" + + results = _("Unexpected Error occured: No QRCODE generated") + status = HTTPStatus.OK + enketo_urls = get_enketo_urls(formhub_url, id_string) + if enketo_urls: + url = enketo_urls.get("url") + image = generate_qrcode(url) + results = f"""{url} +
    {url}""" - results = _(u"Unexpected Error occured: No QRCODE generated") - status = 200 - try: - enketo_urls = get_enketo_urls(formhub_url, id_string) - except Exception as e: - error_msg = _(u"Error Generating QRCODE: %s" % e) - results = """
    %s
    """ % error_msg - status = 400 else: - if enketo_urls: - url = enketo_urls.get("url") - image = generate_qrcode(url) - results = """%s -
    %s""" \ - % (image, url, url, url) - else: - status = 400 + status = HTTPStatus.BAD_REQUEST - return HttpResponse(results, content_type='text/html', status=status) + return HttpResponse(results, content_type="text/html", status=status) def enketo_preview(request, username, id_string): - xform = get_form({ - 'user__username__iexact': username, - 'id_string__iexact': id_string - }) + """Redirects a user to the Enketo preview URL for the given form ``id_string``.""" + xform = get_form( + {"user__username__iexact": username, "id_string__iexact": id_string} + ) owner = xform.user if not has_permission(xform, owner, request, xform.shared): - return HttpResponseForbidden(_(u'Not shared.')) + return HttpResponseForbidden(_("Not shared.")) try: - enketo_urls = get_enketo_urls( - xform.url, xform.id_string) + enketo_urls = get_enketo_urls(xform.url, xform.id_string) - enketo_preview_url = enketo_urls.get('preview_url') + enketo_preview_url = enketo_urls.get("preview_url") except EnketoError as e: return HttpResponse(e) @@ -1434,41 +1518,44 @@ def service_health(request): service_statuses = {} # Check if Database connections are present & data is retrievable - for database in getattr(settings, 'DATABASES').keys(): + for database in getattr(settings, "DATABASES").keys(): + # pylint: disable=broad-except try: XForm.objects.using(database).first() except Exception as e: - service_statuses[f'{database}-Database'] = f'Degraded state; {e}' + service_statuses[f"{database}-Database"] = f"Degraded state; {e}" service_degraded = True else: - service_statuses[f'{database}-Database'] = 'OK' + service_statuses[f"{database}-Database"] = "OK" # Check if cache is accessible + # pylint: disable=broad-except try: - cache.set('ping', 'pong') - cache.delete('ping') + cache.set("ping", "pong") + cache.delete("ping") except Exception as e: - service_statuses['Cache-Service'] = f'Degraded state; {e}' + service_statuses["Cache-Service"] = f"Degraded state; {e}" else: - service_statuses['Cache-Service'] = 'OK' + service_statuses["Cache-Service"] = "OK" - return HttpResponse( - json.dumps(service_statuses), - status=500 if service_degraded else 200, - content_type='application/json') + return JsonResponse( + service_statuses, + status=HTTPStatus.INTERNAL_SERVER_ERROR if service_degraded else HTTPStatus.OK, + ) @require_GET @login_required def username_list(request): data = [] - query = request.GET.get('query', None) + query = request.GET.get("query", None) if query: - users = User.objects.values('username')\ - .filter(username__startswith=query, is_active=True, pk__gte=0) - data = [user['username'] for user in users] + users = User.objects.values("username").filter( + username__startswith=query, is_active=True, pk__gte=0 + ) + data = [user["username"] for user in users] - return HttpResponse(json.dumps(data), content_type='application/json') + return JsonResponse(data) class OnaAuthorizationView(AuthorizationView): @@ -1479,7 +1566,7 @@ class OnaAuthorizationView(AuthorizationView): """ def get_context_data(self, **kwargs): - context = super(OnaAuthorizationView, self).get_context_data(**kwargs) - context['user'] = self.request.user - context['request_path'] = self.request.get_full_path() + context = super().get_context_data(**kwargs) + context["user"] = self.request.user + context["request_path"] = self.request.get_full_path() return context From 2e12885a6f9699c99605e10d3a7b8ebbf9fa2407 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 22:42:51 +0300 Subject: [PATCH 112/234] main.views: cleanup fixes --- onadata/apps/main/views.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index ed24267772..2d1c0589a0 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -368,7 +368,9 @@ def profile_settings(request, username): reverse(public_profile, kwargs={"username": request.user.username}) ) else: - form = UserProfileForm(instance=profile, initial={"email": content_user.email}) + form = UserProfileForm( + instance=user_profile, initial={"email": content_user.email} + ) return render( request, "settings.html", {"content_user": content_user, "form": form} @@ -611,7 +613,7 @@ def api(request, username=None, id_string=None): response_text = f"{callback}({response_text})" response = HttpResponse(response_text) else: - response = JsonResponse(list(cursor)) + response = JsonResponse(list(cursor), safe=False) add_cors_headers(response) @@ -1475,15 +1477,21 @@ def qrcode(request, username, id_string): results = _("Unexpected Error occured: No QRCODE generated") status = HTTPStatus.OK - enketo_urls = get_enketo_urls(formhub_url, id_string) - if enketo_urls: - url = enketo_urls.get("url") - image = generate_qrcode(url) - results = f"""{url} -
    {url}""" - - else: + try: + enketo_urls = get_enketo_urls(formhub_url, id_string) + except EnketoError as e: + error_msg = _(f"Error Generating QRCODE: {e}") + results = f"""
    {error_msg}
    """ status = HTTPStatus.BAD_REQUEST + else: + if enketo_urls: + url = enketo_urls.get("url") + image = generate_qrcode(url) + results = f"""{url} +
    {url}""" + + else: + status = HTTPStatus.BAD_REQUEST return HttpResponse(results, content_type="text/html", status=status) From 52e5a6632dec9057c78266bc1792455c7bb14c86 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 22:55:03 +0300 Subject: [PATCH 113/234] Add some debug statements in CI only failing test. --- onadata/apps/logger/tests/test_briefcase_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 14d8603f77..7c859155c6 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -25,6 +25,7 @@ @urlmatch(netloc=r"(.*\.)?testserver$") def form_list_xml(url, request, **kwargs): + """Mock different ODK Aggregate Server API requests.""" response = requests.Response() factory = RequestFactory() req = factory.get(url.path) @@ -43,16 +44,20 @@ def form_list_xml(url, request, **kwargs): res = download_media_data( req, username="bob", id_string=id_string, data_id=data_id ) + assert res.status_code == 200, f"{data_id} - {res.content} {res.status_code}" + # pylint: disable=protected-access response._content = get_streaming_content(res) else: res = formList(req, username="bob") response.status_code = 200 if not response._content: + # pylint: disable=protected-access response._content = res.content return response def get_streaming_content(res): + """Return the contents of ``res.streaming_content``.""" tmp = BytesIO() for chunk in res.streaming_content: tmp.write(chunk) From 9ac1b14b030abf3d6f52b7517eaf3aac7786480b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 23:33:27 +0300 Subject: [PATCH 114/234] Use defusedxml for parsing xml in osm module defusedxml is considered more secure --- onadata/libs/utils/osm.py | 6 ++++-- requirements/base.pip | 27 ++++++++++++++------------- setup.cfg | 2 ++ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 45601a9877..7c04bb14de 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -9,8 +9,10 @@ from django.contrib.gis.geos import GeometryCollection, LineString, Point, Polygon from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from six import iteritems + +from defusedxml.lxml import fromstring from lxml import etree +from six import iteritems from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance @@ -23,7 +25,7 @@ def _get_xml_obj(xml): if not isinstance(xml, bytes): xml = xml.strip().encode() try: - return etree.fromstring(xml) # pylint: disable=no-member + return fromstring(xml) except etree.XMLSyntaxError as e: # pylint: disable=no-member if "Attribute action redefined" in e.msg: xml = xml.replace(b'action="modify" ', b"") diff --git a/requirements/base.pip b/requirements/base.pip index 2c4756f624..0115581198 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -26,7 +26,7 @@ analytics-python==1.4.0 # via onadata appoptics-metrics==5.1.0 # via onadata -asgiref==3.5.0 +asgiref==3.5.1 # via django async-timeout==4.0.2 # via redis @@ -40,9 +40,9 @@ backoff==1.10.0 # via analytics-python billiard==3.6.4.0 # via celery -boto3==1.21.46 +boto3==1.22.7 # via tabulator -botocore==1.24.46 +botocore==1.25.7 # via # boto3 # s3transfer @@ -60,7 +60,7 @@ chardet==4.0.0 # tabulator charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via # celery # click-didyoumean @@ -75,7 +75,7 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery -cryptography==36.0.2 +cryptography==37.0.2 # via # jwcrypto # onadata @@ -85,6 +85,7 @@ datapackage==1.15.2 defusedxml==0.7.1 # via # djangorestframework-xml + # onadata # pyxform deprecated==1.2.13 # via @@ -114,7 +115,7 @@ django-activity-stream==1.4.0 # via onadata django-cors-headers==3.11.0 # via onadata -django-debug-toolbar==3.2.4 +django-debug-toolbar==3.4.0 # via onadata django-filter==21.1 # via onadata @@ -124,7 +125,7 @@ django-guardian==2.4.0 # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==1.7.1 +django-oauth-toolkit==2.0.0 # via onadata django-ordered-model==3.5 # via onadata @@ -138,7 +139,7 @@ django-render-block==0.9.1 # via django-templated-email django-reversion==5.0.0 # via onadata -django-taggit==2.1.0 +django-taggit==3.0.0 # via onadata django-templated-email==3.0.0 # via onadata @@ -196,7 +197,7 @@ inflection==0.5.1 # via djangorestframework-jsonapi isodate==0.6.1 # via tableschema -jinja2==3.1.1 +jinja2==3.1.2 # via sphinx jmespath==1.0.0 # via @@ -212,7 +213,7 @@ jsonschema==4.4.0 # via # datapackage # tableschema -jwcrypto==1.0 +jwcrypto==1.2 # via django-oauth-toolkit kombu==5.2.4 # via celery @@ -270,7 +271,7 @@ pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 -pygments==2.11.2 +pygments==2.12.0 # via sphinx pyjwt[crypto]==2.3.0 # via @@ -371,7 +372,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy==1.4.35 +sqlalchemy==1.4.36 # via tabulator sqlparse==0.4.2 # via @@ -403,7 +404,7 @@ vine==5.0.0 # kombu wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.14.0 +wrapt==1.14.1 # via deprecated xlrd==2.0.1 # via diff --git a/setup.cfg b/setup.cfg index bb0bae1485..c63d198bdf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -103,6 +103,8 @@ install_requires = deprecated # Redis cache django-redis + # osm + defusedxml python_requires = >= 3.8 setup_requires = setuptools_scm From 68a40b369f1e1c8b137d29291139330d84d55e9e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 4 May 2022 23:42:12 +0300 Subject: [PATCH 115/234] Add some debug statements in CI only failing test. --- onadata/apps/logger/tests/test_briefcase_client.py | 5 ++++- onadata/apps/main/views.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 7c859155c6..60c595aa35 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -44,7 +44,10 @@ def form_list_xml(url, request, **kwargs): res = download_media_data( req, username="bob", id_string=id_string, data_id=data_id ) - assert res.status_code == 200, f"{data_id} - {res.content} {res.status_code}" + ids = list(Instance.objects.values_list("id", flat=True)) + assert ( + res.status_code == 200 + ), f"{data_id} - {res.content} {res.status_code} -{ids}" # pylint: disable=protected-access response._content = get_streaming_content(res) else: diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 2d1c0589a0..116c1cbbca 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -1137,7 +1137,7 @@ def download_media_data(request, username, id_string, data_id): request.user, owner, _( - "Media f'{os.path.basename(file_path)}' " + f"Media '{os.path.basename(file_path)}' " f"downloaded from '{xform.id_string}'." ), audit, From d096758a7ef88a888b695d530c516ba628ccd683 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 00:01:59 +0300 Subject: [PATCH 116/234] Import etree via defusedxml module --- onadata/libs/utils/osm.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 7c04bb14de..925519821e 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -10,8 +10,7 @@ from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from defusedxml.lxml import fromstring -from lxml import etree +from defusedxml.lxml import _etree, fromstring from six import iteritems from onadata.apps.logger.models.attachment import Attachment @@ -26,7 +25,7 @@ def _get_xml_obj(xml): xml = xml.strip().encode() try: return fromstring(xml) - except etree.XMLSyntaxError as e: # pylint: disable=no-member + except _etree.XMLSyntaxError as e: # pylint: disable=no-member if "Attribute action redefined" in e.msg: xml = xml.replace(b'action="modify" ', b"") From d14cfa7f91835ec19de0d30404a9612ccd7abacd Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 00:32:52 +0300 Subject: [PATCH 117/234] batch: cleanup --- .../apps/api/viewsets/user_profile_viewset.py | 120 ++++++++++-------- onadata/libs/utils/dict_tools.py | 7 +- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 71f8b9a364..c4eccf66b4 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -5,18 +5,17 @@ import datetime import json -from six.moves.urllib.parse import urlencode - from django.conf import settings from django.core.cache import cache from django.core.validators import ValidationError from django.db.models import Count -from django.http import HttpResponseRedirect, HttpResponseBadRequest -from django.utils.translation import ugettext as _ +from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.utils import timezone from django.utils.module_loading import import_string +from django.utils.translation import ugettext as _ +from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import serializers, status from rest_framework.decorators import action @@ -25,22 +24,14 @@ from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from multidb.pinning import use_master +from six.moves.urllib.parse import urlencode -from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.permissions import UserProfilePermissions +from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.instance import Instance from onadata.apps.main.models import UserProfile -from onadata.libs.utils.email import get_verification_email_data, get_verification_url -from onadata.libs.utils.cache_tools import ( - safe_delete, - CHANGE_PASSWORD_ATTEMPTS, - LOCKOUT_CHANGE_PASSWORD_USER, - USER_PROFILE_PREFIX, -) from onadata.libs import filters -from onadata.libs.utils.user_auth import invalidate_and_regen_tokens from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin @@ -49,6 +40,14 @@ MonthlySubmissionsSerializer, ) from onadata.libs.serializers.user_profile_serializer import UserProfileSerializer +from onadata.libs.utils.cache_tools import ( + CHANGE_PASSWORD_ATTEMPTS, + LOCKOUT_CHANGE_PASSWORD_USER, + USER_PROFILE_PREFIX, + safe_delete, +) +from onadata.libs.utils.email import get_verification_email_data, get_verification_url +from onadata.libs.utils.user_auth import invalidate_and_regen_tokens BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name LOCKOUT_TIME = getattr(settings, "LOCKOUT_TIME", 1800) @@ -78,9 +77,9 @@ def check_if_key_exists(a_key, expected_dict): for key, value in expected_dict.items(): if key == a_key: return True - elif isinstance(value, dict): + if isinstance(value, dict): return check_if_key_exists(a_key, value) - elif isinstance(value, list): + if isinstance(value, list): for list_item in value: if isinstance(list_item, dict): return check_if_key_exists(a_key, list_item) @@ -99,30 +98,35 @@ def serializer_from_settings(): def set_is_email_verified(profile, is_email_verified): + """Sets is_email_verified value in the profile's metadata object.""" profile.metadata.update({"is_email_verified": is_email_verified}) profile.save() def check_user_lockout(request): + """Returns the error object with lockout error message.""" username = request.user.username - lockout = cache.get("{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username)) - response_obj = { - "error": "Too many password reset attempts, Try again in {} minutes" - } + lockout = cache.get(f"{LOCKOUT_CHANGE_PASSWORD_USER}{username}") if lockout: time_locked_out = datetime.datetime.now() - datetime.datetime.strptime( lockout, "%Y-%m-%dT%H:%M:%S" ) remaining_time = round((LOCKOUT_TIME - time_locked_out.seconds) / 60) - response = response_obj["error"].format(remaining_time) - return response + response_obj = { + "error": _( + "Too many password reset attempts. " + f"Try again in {remaining_time} minutes" + ) + } + return response_obj + return None def change_password_attempts(request): """Track number of login attempts made by user within a specified amount of time""" username = request.user.username - password_attempts = "{}{}".format(CHANGE_PASSWORD_ATTEMPTS, username) + password_attempts = f"{CHANGE_PASSWORD_ATTEMPTS}{username}" attempts = cache.get(password_attempts) if attempts: @@ -130,7 +134,7 @@ def change_password_attempts(request): attempts = cache.get(password_attempts) if attempts >= MAX_CHANGE_PASSWORD_ATTEMPTS: cache.set( - "{}{}".format(LOCKOUT_CHANGE_PASSWORD_USER, username), + f"{LOCKOUT_CHANGE_PASSWORD_USER}{username}", datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), LOCKOUT_TIME, ) @@ -144,8 +148,9 @@ def change_password_attempts(request): return 1 +# pylint: disable=too-many-ancestors class UserProfileViewSet( - AuthenticateHeaderMixin, # pylint: disable=R0901 + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, ObjectLookupMixin, @@ -170,7 +175,7 @@ class UserProfileViewSet( def get_object(self, queryset=None): """Lookup user profile by pk or username""" if self.kwargs.get(self.lookup_field, None) is None: - raise ParseError("Expected URL keyword argument `%s`." % self.lookup_field) + raise ParseError(_(f"Expected URL keyword argument `{self.lookup_field}`.")) if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -181,7 +186,7 @@ def get_object(self, queryset=None): if self.lookup_field in serializer.get_fields(): k = serializer.get_fields()[self.lookup_field] if isinstance(k, serializers.HyperlinkedRelatedField): - lookup_field = "%s__%s" % (self.lookup_field, k.lookup_field) + lookup_field = f"{self.lookup_field}__{k.lookup_field}" lookup = self.kwargs[self.lookup_field] filter_kwargs = {lookup_field: lookup} @@ -189,7 +194,7 @@ def get_object(self, queryset=None): try: user_pk = int(lookup) except (TypeError, ValueError): - filter_kwargs = {"%s__iexact" % lookup_field: lookup} + filter_kwargs = {f"{lookup_field}__iexact": lookup} else: filter_kwargs = {"user__pk": user_pk} @@ -203,7 +208,7 @@ def get_object(self, queryset=None): def update(self, request, *args, **kwargs): """Update user in cache and db""" username = kwargs.get("user") - response = super(UserProfileViewSet, self).update(request, *args, **kwargs) + response = super().update(request, *args, **kwargs) cache.set(f"{USER_PROFILE_PREFIX}{username}", response.data) return response @@ -213,12 +218,12 @@ def retrieve(self, request, *args, **kwargs): cached_user = cache.get(f"{USER_PROFILE_PREFIX}{username}") if cached_user: return Response(cached_user) - response = super(UserProfileViewSet, self).retrieve(request, *args, **kwargs) + response = super().retrieve(request, *args, **kwargs) return response def create(self, request, *args, **kwargs): """Create and cache user profile""" - response = super(UserProfileViewSet, self).create(request, *args, **kwargs) + response = super().create(request, *args, **kwargs) profile = response.data user_name = profile.get("username") cache.set(f"{USER_PROFILE_PREFIX}{user_name}", profile) @@ -235,7 +240,6 @@ def change_password(self, request, *args, **kwargs): # noqa current_password = request.data.get("current_password", None) new_password = request.data.get("new_password", None) lock_out = check_user_lockout(request) - response_obj = {"error": "Invalid password. You have {} attempts left."} if new_password: if not lock_out: @@ -254,8 +258,13 @@ def change_password(self, request, *args, **kwargs): # noqa response = change_password_attempts(request) if isinstance(response, int): limits_remaining = MAX_CHANGE_PASSWORD_ATTEMPTS - response - response = response_obj["error"].format(limits_remaining) - return Response(data=response, status=status.HTTP_400_BAD_REQUEST) + response_obj = { + "error": _( + "Invalid password. " + f"You have {limits_remaining} attempts left." + ) + } + return Response(data=response_obj, status=status.HTTP_400_BAD_REQUEST) return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) @@ -278,7 +287,7 @@ def partial_update(self, request, *args, **kwargs): profile.save() return Response(data=profile.metadata, status=status.HTTP_200_OK) - return super(UserProfileViewSet, self).partial_update(request, *args, **kwargs) + return super().partial_update(request, *args, **kwargs) @action(methods=["GET"], detail=True) def monthly_submissions(self, request, *args, **kwargs): @@ -317,6 +326,7 @@ def monthly_submissions(self, request, *args, **kwargs): serializer = MonthlySubmissionsSerializer(instance_count, many=True) return Response(serializer.data[0]) + # pylint: disable=no-self-use @action(detail=False) def verify_email(self, request, *args, **kwargs): verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) @@ -328,26 +338,28 @@ def verify_email(self, request, *args, **kwargs): verification_key = request.query_params.get("verification_key") response_message = _("Missing or invalid verification key") if verification_key: - rp = None + registration_profile = None try: - rp = RegistrationProfile.objects.select_related( + registration_profile = RegistrationProfile.objects.select_related( "user", "user__profile" ).get(activation_key=verification_key) except RegistrationProfile.DoesNotExist: with use_master: try: - rp = RegistrationProfile.objects.select_related( - "user", "user__profile" - ).get(activation_key=verification_key) + registration_profile = ( + RegistrationProfile.objects.select_related( + "user", "user__profile" + ).get(activation_key=verification_key) + ) except RegistrationProfile.DoesNotExist: pass - if rp: - rp.activation_key = verified_key_text - rp.save() + if registration_profile: + registration_profile.activation_key = verified_key_text + registration_profile.save() - username = rp.user.username - set_is_email_verified(rp.user.profile, True) + username = registration_profile.user.username + set_is_email_verified(registration_profile.user.profile, True) # Clear profiles cache safe_delete(f"{USER_PROFILE_PREFIX}{username}") @@ -355,7 +367,7 @@ def verify_email(self, request, *args, **kwargs): if redirect_url: query_params_string = urlencode(response_data) - redirect_url = "{}?{}".format(redirect_url, query_params_string) + redirect_url = f"{redirect_url}?{query_params_string}" return HttpResponseRedirect(redirect_url) @@ -363,6 +375,7 @@ def verify_email(self, request, *args, **kwargs): return HttpResponseBadRequest(response_message) + # pylint: disable=no-self-use @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) @@ -375,16 +388,18 @@ def send_verification_email(self, request, *args, **kwargs): if username: try: - rp = RegistrationProfile.objects.get(user__username=username) + registration_profile = RegistrationProfile.objects.get( + user__username=username + ) except RegistrationProfile.DoesNotExist: pass else: - set_is_email_verified(rp.user.profile, False) + set_is_email_verified(registration_profile.user.profile, False) - verification_key = rp.activation_key + verification_key = registration_profile.activation_key if verification_key == verified_key_text: verification_key = ( - rp.user.registrationprofile.create_new_activation_key() + registration_profile.user.registrationprofile.create_new_activation_key() ) verification_url = get_verification_url( @@ -392,7 +407,10 @@ def send_verification_email(self, request, *args, **kwargs): ) email_data = get_verification_email_data( - rp.user.email, rp.user.username, verification_url, request + registration_profile.user.email, + registration_profile.user.username, + verification_url, + request, ) send_verification_email.delay(**email_data) diff --git a/onadata/libs/utils/dict_tools.py b/onadata/libs/utils/dict_tools.py index 466ae6820f..a348bf26aa 100644 --- a/onadata/libs/utils/dict_tools.py +++ b/onadata/libs/utils/dict_tools.py @@ -53,6 +53,7 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None): """ result = {} + # pylint: disable=too-many-nested-blocks for row in list_of_dicts: for k, v in row.items(): if isinstance(v, list): @@ -98,7 +99,7 @@ def remove_indices_from_dict(obj): Removes indices from a obj dict. """ if not isinstance(obj, dict): - raise ValueError("Expecting a dict, found: {}".format(type(obj))) + raise ValueError(f"Expecting a dict, found: {type(obj)}") result = {} for key, val in obj.items(): @@ -150,7 +151,7 @@ def dict_lists2strings(adict): :param d: The dict to convert. :returns: The converted dict.""" for k, v in adict.items(): - if isinstance(v, list) and all([isinstance(e, str) for e in v]): + if isinstance(v, list) and all(isinstance(e, str) for e in v): adict[k] = " ".join(v) elif isinstance(v, dict): adict[k] = dict_lists2strings(v) @@ -182,7 +183,7 @@ def query_list_to_dict(query_list_str): Returns a 'label' and 'text' from a Rapidpro values JSON string as a dict. """ data_list = json.loads(query_list_str) - data_dict = dict() + data_dict = {} for value in data_list: data_dict[value["label"]] = value["text"] From 12a5ba1411417fb8cfeb7fa510263a00c060c4cb Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 00:40:22 +0300 Subject: [PATCH 118/234] Use defusedxml tostring import --- onadata/libs/utils/osm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 925519821e..cfad61280e 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -10,7 +10,7 @@ from django.contrib.gis.geos.error import GEOSException from django.db import IntegrityError, models, transaction -from defusedxml.lxml import _etree, fromstring +from defusedxml.lxml import _etree, fromstring, tostring from six import iteritems from onadata.apps.logger.models.attachment import Attachment @@ -64,7 +64,7 @@ def get_combined_osm(osm_list): osm.append(child) if osm is not None: # pylint: disable=no-member - return etree.tostring(osm, encoding="utf-8", xml_declaration=True) + return tostring(osm, encoding="utf-8", xml_declaration=True) if isinstance(osm_list, dict): if "detail" in osm_list: xml = f"{osm_list['detail']}" From a8110c4ef69fa9dc2b474d6d7be48066c2efef77 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 00:45:34 +0300 Subject: [PATCH 119/234] user_profile_viewset.py: cleanup --- onadata/apps/api/viewsets/user_profile_viewset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index c4eccf66b4..640f6b4baa 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -15,6 +15,8 @@ from django.utils.module_loading import import_string from django.utils.translation import ugettext as _ +from six.moves.urllib.parse import urlencode + from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import serializers, status @@ -24,7 +26,6 @@ from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from six.moves.urllib.parse import urlencode from onadata.apps.api.permissions import UserProfilePermissions from onadata.apps.api.tasks import send_verification_email @@ -269,6 +270,7 @@ def change_password(self, request, *args, **kwargs): # noqa return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) def partial_update(self, request, *args, **kwargs): + """Allows for partial update of the user profile data.""" profile = self.get_object() metadata = profile.metadata or {} if request.data.get("overwrite") == "false": @@ -329,6 +331,7 @@ def monthly_submissions(self, request, *args, **kwargs): # pylint: disable=no-self-use @action(detail=False) def verify_email(self, request, *args, **kwargs): + """Accpet's email verification token and marks the profile as verified.""" verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: @@ -378,6 +381,7 @@ def verify_email(self, request, *args, **kwargs): # pylint: disable=no-self-use @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): + """Sends verification email on user profile registration.""" verified_key_text = getattr(settings, "VERIFIED_KEY_TEXT", None) if not verified_key_text: return Response(status=status.HTTP_204_NO_CONTENT) From 5af578e68a22d76761dd651e3786401260b03952 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 01:42:36 +0300 Subject: [PATCH 120/234] Add some debug statements in CI only failing test. --- onadata/apps/logger/tests/test_briefcase_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 60c595aa35..2bf42c9663 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -45,9 +45,10 @@ def form_list_xml(url, request, **kwargs): req, username="bob", id_string=id_string, data_id=data_id ) ids = list(Instance.objects.values_list("id", flat=True)) + xids = list(XForm.objects.values_list("id", flat=True)) assert ( res.status_code == 200 - ), f"{data_id} - {res.content} {res.status_code} -{ids}" + ), f"{data_id} - {res.content} {res.status_code} -{ids} {xids} {url}" # pylint: disable=protected-access response._content = get_streaming_content(res) else: From df87d542fb4ae1f5bbeddd92c2d494be048ce229 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 03:23:26 +0300 Subject: [PATCH 121/234] batch: cleanup --- .../tests/viewsets/test_user_profile_viewset.py | 14 +++++++++----- onadata/apps/api/viewsets/briefcase_viewset.py | 3 +++ onadata/apps/logger/models/xform.py | 6 ++---- onadata/apps/main/views.py | 15 +++++++++++---- .../libs/serializers/password_reset_serializer.py | 1 + onadata/libs/utils/export_builder.py | 8 ++++---- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index 265b201d7b..19fdf2b9de 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -791,7 +791,9 @@ def test_change_password_wrong_current_password(self): response = view(request, user="bob") user = User.objects.get(username__iexact=self.user.username) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, "Invalid password. You have 9 attempts left.") + self.assertEqual( + response.data, {"error": "Invalid password. You have 9 attempts left."} + ) self.assertFalse(user.check_password(new_password)) def test_profile_create_with_name(self): @@ -1125,7 +1127,7 @@ def grant_perms_form_builder( setattr( response, "_content", - {"detail": "Successfully granted default model level perms to" " user."}, + {"detail": "Successfully granted default model level perms to user."}, ) return response @@ -1318,7 +1320,7 @@ def test_change_password_attempts(self): response = view(request, user="bob") self.assertEqual(response.status_code, 400) self.assertEqual( - response.data, "Invalid password." " You have 9 attempts left." + response.data, {"error": "Invalid password. You have 9 attempts left."} ) self.assertEqual(cache.get("change_password_attempts-bob"), 1) @@ -1326,7 +1328,9 @@ def test_change_password_attempts(self): request = self.factory.post("/", data=post_data, **self.extra) response = view(request, user="bob") self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, "Invalid password. You have 8 attempts left.") + self.assertEqual( + response.data, {"error": "Invalid password. You have 8 attempts left."} + ) self.assertEqual(cache.get("change_password_attempts-bob"), 2) # check user is locked out @@ -1337,7 +1341,7 @@ def test_change_password_attempts(self): self.assertEqual(response.status_code, 400) self.assertEqual( response.data, - "Too many password reset attempts," " Try again in 30 minutes", + {"error": "Too many password reset attempts. Try again in 30 minutes"}, ) self.assertEqual(cache.get("change_password_attempts-bob"), 10) self.assertIsNotNone(cache.get("lockout_change_password_user-bob")) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 6770d81406..2472adcdf0 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -37,6 +37,7 @@ from onadata.libs.utils.logger_tools import PublishXForm, publish_form from onadata.libs.utils.viewer_tools import get_form +# pylint: disable=invalid-name User = get_user_model() @@ -92,6 +93,7 @@ class BriefcaseViewset( # pylint: disable=unused-argument def get_object(self, queryset=None): + """Returns an Instance submission object for the given UUID.""" form_id = self.request.GET.get("formId", "") id_string = _extract_id_string(form_id) uuid = _extract_uuid(form_id) @@ -109,6 +111,7 @@ def get_object(self, queryset=None): # pylint: disable=too-many-branches def filter_queryset(self, queryset): + """Filters an XForm submission instances using ODK Aggregate query parameters.""" username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index c42fa7d409..32b663c75b 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -210,9 +210,7 @@ def check_version_set(survey): if isinstance(survey_json, str): survey = builder.create_survey_element_from_json(survey_json) elif isinstance(survey_json, dict): - survey = builder.create_survey_element_from_dict( - survey_json - ) + survey = builder.create_survey_element_from_dict(survey_json) return survey @@ -708,7 +706,7 @@ def _expand_geocodes(self, item, key, elem): if elem and elem.bind.get("type") == "geopoint": geodata = item[key].split() for i, v in enumerate(geodata): - new_key = f"{key}_{self.geodata_suffixes[i]}" + new_key = f"{key}_{self.GEODATA_SUFFIXES[i]}" item[new_key] = v def get_data_for_excel(self): diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 116c1cbbca..ce79411e1a 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -126,7 +126,7 @@ def set_form(): id_string__iexact=id_string, deleted_at__isnull=True, ) - if len(id_string) > 0 and id_string[0].isdigit(): + if id_string and id_string[0].isdigit(): id_string = "_" + id_string path = xform.xls.name if default_storage.exists(path): @@ -206,6 +206,7 @@ def profile(request, username): if request.method == "POST" and request.user.is_authenticated: def set_form(): + """Publishes the XLSForm.""" form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user).survey audit = {} @@ -1354,6 +1355,7 @@ def update_xform(request, username, id_string): owner = xform.user def set_form(): + """Publishes the XLSForm""" form = QuickConverter(request.POST, request.FILES) survey = form.publish(request.user, id_string).survey enketo_webform_url = reverse( @@ -1416,7 +1418,7 @@ def activity_fields(request): {"id": "msg", "label": "Description", "type": "string", "searchable": True}, ] - return JsonResponse(fields) + return JsonResponse(fields, safe=False) @is_owner @@ -1458,7 +1460,9 @@ def stringify_unknowns(obj): response_text = f"{callback}({response_text})" return HttpResponse(response_text) - return JsonResponse(records, json_dumps_params={"default": stringify_unknowns}) + return JsonResponse( + records, json_dumps_params={"default": stringify_unknowns}, safe=False + ) def qrcode(request, username, id_string): @@ -1555,6 +1559,7 @@ def service_health(request): @require_GET @login_required def username_list(request): + """Show's the list of usernames.""" data = [] query = request.GET.get("query", None) if query: @@ -1563,9 +1568,10 @@ def username_list(request): ) data = [user["username"] for user in users] - return JsonResponse(data) + return JsonResponse(data, safe=False) +# pylint: disable=too-few-public-methods class OnaAuthorizationView(AuthorizationView): """ @@ -1574,6 +1580,7 @@ class OnaAuthorizationView(AuthorizationView): """ def get_context_data(self, **kwargs): + """Adds `user` and `request_path` to context and returns the context.""" context = super().get_context_data(**kwargs) context["user"] = self.request.user context["request_path"] = self.request.get_full_path() diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index 9b8e9d0271..c163c6b4fe 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -234,6 +234,7 @@ def validate(self, attrs): return attrs def create(self, validated_data, instance=None): + """Set a new user password and invalidate/regenerate tokens.""" instance = PasswordResetChange(**validated_data) instance.save() diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 7d916b1cf5..f2538d5a9a 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -195,7 +195,7 @@ def encode_if_str(row, key, encode_dates=False, sav_writer=None): if isinstance(val, (datetime, date)): if sav_writer: if isinstance(val, datetime): - if len(val.isoformat()): + if val.isoformat(): strptime_fmt = "%Y-%m-%dT%H:%M:%S" else: strptime_fmt = "%Y-%m-%dT%H:%M:%S.%f%z" @@ -983,7 +983,7 @@ def pre_process_row(self, row, section): return row - # pylint: disable=too-many-locals,too-many-branches + # pylint: disable=too-many-locals,too-many-branches,unused-argument def to_zipped_csv(self, path, data, *args, **kwargs): """Export CSV formatted files from ``data`` and zip the files.""" @@ -1101,7 +1101,7 @@ def get_valid_sheet_name(cls, desired_name, existing_names): i += 1 return generated_name - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals,too-many-statements,unused-argument def to_xls_export(self, path, data, *args, **kwargs): """Export data to a spreadsheet document.""" @@ -1385,7 +1385,7 @@ def _is_numeric(xpath, element_type, data_dictionary): duplicate_names = [] # list of (xpath, var_name) already_done = [] # list of xpaths - for field, label, xpath, var_name in fields_and_labels: + for _field, label, xpath, var_name in fields_and_labels: var_labels[var_name] = label # keep track of duplicates if xpath not in already_done: From 312481966c03d1935fa767cb372f08c3a95ea45b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 03:54:30 +0300 Subject: [PATCH 122/234] batch: cleanup --- .../management/commands/update_enketo_urls.py | 83 +++++++++++-------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 96cc078c7a..1611a1e722 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -1,54 +1,66 @@ +# -*- coding: utf-8 -*- +"""update_enketo_urls - command to update Enketo preview URLs in the MetaData model.""" +import os + from django.core.management.base import BaseCommand, CommandError from django.db.models import Q from django.http import HttpRequest from django.utils.translation import ugettext_lazy from onadata.apps.main.models.meta_data import MetaData -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url class Command(BaseCommand): + """Updates enketo preview urls in MetaData model""" + help = ugettext_lazy("Updates enketo preview urls in MetaData model") def add_arguments(self, parser): parser.add_argument( - "-n", "--server_name", dest="server_name", default="enketo.ona.io") - parser.add_argument( - "-p", "--server_port", dest="server_port", default="443") - parser.add_argument( - "-r", "--protocol", dest="protocol", default="https") + "-n", "--server_name", dest="server_name", default="enketo.ona.io" + ) + parser.add_argument("-p", "--server_port", dest="server_port", default="443") + parser.add_argument("-r", "--protocol", dest="protocol", default="https") parser.add_argument( - "-c", "--generate_consistent_urls", - dest="generate_consistent_urls", default=True) + "-c", + "--generate_consistent_urls", + dest="generate_consistent_urls", + default=True, + ) + # pylint: disable=too-many-locals def handle(self, *args, **options): + """Updates enketo preview urls in MetaData model""" request = HttpRequest() - server_name = options.get('server_name') - server_port = options.get('server_port') - protocol = options.get('protocol') - generate_consistent_urls = options.get('generate_consistent_urls') + server_name = options.get("server_name") + server_port = options.get("server_port") + protocol = options.get("protocol") + generate_consistent_urls = options.get("generate_consistent_urls") if not server_name or not server_port or not protocol: raise CommandError( - 'please provide a server_name, a server_port and a protocol') + "please provide a server_name, a server_port and a protocol" + ) - if server_name not in ['ona.io', 'stage.ona.io', 'localhost']: - raise CommandError('server name provided is not valid') + if server_name not in ["ona.io", "stage.ona.io", "localhost"]: + raise CommandError("server name provided is not valid") - if protocol not in ['http', 'https']: - raise CommandError('protocol provided is not valid') + if protocol not in ["http", "https"]: + raise CommandError("protocol provided is not valid") # required for generation of enketo url - request.META['HTTP_HOST'] = '%s:%s' % (server_name, server_port)\ - if server_port != '80' else server_name + request.META["HTTP_HOST"] = ( + f"{server_name}:{server_port}" if server_port != "80" else server_name + ) # required for generation of enketo preview url - request.META['SERVER_NAME'] = server_name - request.META['SERVER_PORT'] = server_port + request.META["SERVER_NAME"] = server_name + request.META["SERVER_PORT"] = server_port resultset = MetaData.objects.filter( - Q(data_type='enketo_url') | Q(data_type='enketo_preview_url')) + Q(data_type="enketo_url") | Q(data_type="enketo_preview_url") + ) for meta_data in resultset: username = meta_data.content_object.user.username @@ -58,22 +70,27 @@ def handle(self, *args, **options): data_value = meta_data.data_value xform = meta_data.content_object xform_pk = xform.pk + if not os.path.exists("/tmp/enketo_url"): + break - with open('/tmp/enketo_url', 'a') as f: + with open("/tmp/enketo_url", "a", encoding="utf-8") as f: form_url = get_form_url( - request, username=username, id_string=id_string, + request, + username=username, xform_pk=xform_pk, - generate_consistent_urls=generate_consistent_urls) + generate_consistent_urls=generate_consistent_urls, + ) enketo_urls = get_enketo_urls(form_url, id_string) - if data_type == 'enketo_url': - _enketo_url = (enketo_urls.get('offline_url') or - enketo_urls.get('url')) + if data_type == "enketo_url": + _enketo_url = enketo_urls.get("offline_url") or enketo_urls.get( + "url" + ) MetaData.enketo_url(xform, _enketo_url) - elif data_type == 'enketo_preview_url': - _enketo_preview_url = (enketo_urls.get('preview_url')) + elif data_type == "enketo_preview_url": + _enketo_preview_url = enketo_urls.get("preview_url") MetaData.enketo_preview_url(xform, _enketo_preview_url) - f.write('%s : %s \n' % (id_string, data_value)) - self.stdout.write('%s: %s' % (data_type, meta_data.data_value)) + f.write(f"{id_string} : {data_value} \n") + self.stdout.write(f"{data_type}: {meta_data.data_value}") self.stdout.write("enketo urls update complete!!") From d75de960041a402dbb91132134aa4d93c8cd8568 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 5 May 2022 09:18:03 +0300 Subject: [PATCH 123/234] Dependecies cleanup Signed-off-by: Kipchirchir Sigei --- docker-compose.yml | 4 ++-- docker/onadata-uwsgi/docker-compose.yml | 2 +- requirements/base.pip | 13 ++++++++++--- requirements/s3.in | 2 +- requirements/s3.pip | 8 +++++--- requirements/ses.in | 2 +- requirements/ses.pip | 12 +++++++----- setup.cfg | 6 +++--- 8 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5621ef2347..2ed3c383e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgis/postgis:9.6-3.0 + image: postgis/postgis:13-3.0 environment: - POSTGRES_PASSWORD=onadata - POSTGRES_USER=onadata @@ -29,7 +29,7 @@ services: - db - queue environment: - - SELECTED_PYTHON=python3.6 + - SELECTED_PYTHON=python3.9 - INITDB=false notifications: image: emqx/emqx:4.3.2 diff --git a/docker/onadata-uwsgi/docker-compose.yml b/docker/onadata-uwsgi/docker-compose.yml index d881639e50..27ce6eb30b 100644 --- a/docker/onadata-uwsgi/docker-compose.yml +++ b/docker/onadata-uwsgi/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgis/postgis:9.6-3.0 + image: postgis/postgis:13-3.0 environment: - POSTGRES_PASSWORD=onadata - POSTGRES_USER=onadata diff --git a/requirements/base.pip b/requirements/base.pip index 72b6a4a287..123324ebf8 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -1,10 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/base.pip requirements/base.in # - -e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router @@ -13,6 +12,8 @@ # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in +-e file:///home/kip/src/ona/onadata + # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in -e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest @@ -194,6 +195,10 @@ ijson==3.1.4 # via tabulator imagesize==1.3.0 # via sphinx +importlib-metadata==4.11.3 + # via + # markdown + # sphinx inflection==0.5.1 # via djangorestframework-jsonapi isodate==0.6.1 @@ -414,4 +419,6 @@ xlrd==2.0.1 xlwt==1.3.0 # via onadata xmltodict==0.12.0 - # via onadata \ No newline at end of file + # via onadata +zipp==3.8.0 + # via importlib-metadata diff --git a/requirements/s3.in b/requirements/s3.in index 124bacc622..210fbc7957 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,3 +1,3 @@ django-storages -django >=2.2.20,<3 +django >=3.2.13,<4 boto3 diff --git a/requirements/s3.pip b/requirements/s3.pip index f27f1c9c0c..9f166104d5 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -1,21 +1,23 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/s3.pip requirements/s3.in # +asgiref==3.5.1 + # via django boto3==1.17.74 # via -r requirements/s3.in botocore==1.20.74 # via # boto3 # s3transfer -django-storages==1.11.1 - # via -r requirements/s3.in django==3.2.13 # via # -r requirements/s3.in # django-storages +django-storages==1.11.1 + # via -r requirements/s3.in jmespath==0.10.0 # via # boto3 diff --git a/requirements/ses.in b/requirements/ses.in index 3d549791cd..e23a56f077 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,3 +1,3 @@ boto -django >=2.2.20,<3 +django >=3.2.13,<4 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index 5c2739220e..274e3c3653 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -1,23 +1,25 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --output-file=requirements/ses.pip requirements/ses.in # -boto3==1.17.74 - # via django-ses +asgiref==3.5.1 + # via django boto==2.49.0 # via -r requirements/ses.in +boto3==1.17.74 + # via django-ses botocore==1.20.74 # via # boto3 # s3transfer -django-ses==2.0.0 - # via -r requirements/ses.in django==3.2.13 # via # -r requirements/ses.in # django-ses +django-ses==2.0.0 + # via -r requirements/ses.in future==0.18.2 # via django-ses jmespath==0.10.0 diff --git a/setup.cfg b/setup.cfg index c63d198bdf..be16db420f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ license = Copyright (c) 2022 Ona Systems Inc All rights reserved license_file = LICENSE classifiers = Development Status :: 5 - Production/Stable - Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 project_urls = Documentation = https://api.ona.io/api Source = https://github.com/onaio/onadata @@ -20,7 +20,7 @@ project_urls = packages = find: platforms = any install_requires = - Django>=2.2.20,<4 + Django>=3.2.13,<4 django-guardian django-registration-redux django-templated-email @@ -105,7 +105,7 @@ install_requires = django-redis # osm defusedxml -python_requires = >= 3.8 +python_requires = >= 3.9 setup_requires = setuptools_scm From fadc3d301408151a1a7baff51eac48a491798e8f Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 5 May 2022 10:17:03 +0300 Subject: [PATCH 124/234] Remove local path to onadata repository --- requirements/base.pip | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index 123324ebf8..673d92cdf5 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -12,8 +12,6 @@ # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in --e file:///home/kip/src/ona/onadata - # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in -e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest From 43464c957209b5e915662d23f22ea15fea681f36 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 5 May 2022 10:31:34 +0300 Subject: [PATCH 125/234] Use python3.9 instead of in-built python3 installation --- docker/onadata-uwsgi/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index a62d48b8e0..f989abec35 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -80,11 +80,11 @@ COPY uwsgi.ini /uwsgi.ini # Install service requirements WORKDIR /srv/onadata # hadolint ignore=DL3013 -RUN python3 -m pip install --no-cache-dir -U pip && \ - python3 -m pip install --no-cache-dir -r requirements/base.pip && \ - python3 -m pip install --no-cache-dir -r requirements/s3.pip && \ - python3 -m pip install --no-cache-dir -r requirements/ses.pip && \ - python3 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0 +RUN python3.9 -m pip install --no-cache-dir -U pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/base.pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/s3.pip && \ + python3.9 -m pip install --no-cache-dir -r requirements/ses.pip && \ + python3.9 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0 # Compile API Docs RUN make -C docs html From 9075e79739212706d37e632b0b31368d2baeeeb9 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Thu, 5 May 2022 11:47:33 +0300 Subject: [PATCH 126/234] Deprecation: staticfiles has been renamed to static Signed-off-by: Kipchirchir Sigei --- onadata/libs/templates/rest_framework_swagger/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/templates/rest_framework_swagger/index.html b/onadata/libs/templates/rest_framework_swagger/index.html index c11679bdc0..a12911d154 100644 --- a/onadata/libs/templates/rest_framework_swagger/index.html +++ b/onadata/libs/templates/rest_framework_swagger/index.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} From c780e1e906a91444a0c11f20e79dac3ed6c282da Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 11:50:46 +0300 Subject: [PATCH 127/234] =?UTF-8?q?Fix:=20=1B[200~UnboundLocalError:=20loc?= =?UTF-8?q?al=20variable=20response=5Fobj=20referenced=20before=20assignme?= =?UTF-8?q?nt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- onadata/apps/api/viewsets/user_profile_viewset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index 640f6b4baa..b05559acf2 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -259,13 +259,13 @@ def change_password(self, request, *args, **kwargs): # noqa response = change_password_attempts(request) if isinstance(response, int): limits_remaining = MAX_CHANGE_PASSWORD_ATTEMPTS - response - response_obj = { + response = { "error": _( "Invalid password. " f"You have {limits_remaining} attempts left." ) } - return Response(data=response_obj, status=status.HTTP_400_BAD_REQUEST) + return Response(data=response, status=status.HTTP_400_BAD_REQUEST) return Response(data=lock_out, status=status.HTTP_400_BAD_REQUEST) From d7233a4b203d5f7d89bdacb9512b2f8e326df46c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 13:53:37 +0300 Subject: [PATCH 128/234] Used 'django-upgrade --target-version 3.2' --- onadata/apps/api/admin.py | 15 ++++-- onadata/apps/api/models/odk_token.py | 2 +- onadata/apps/api/tools.py | 2 +- onadata/apps/api/urls/v1_urls.py | 4 +- .../apps/api/viewsets/attachment_viewset.py | 2 +- .../apps/api/viewsets/briefcase_viewset.py | 2 +- onadata/apps/api/viewsets/connect_viewset.py | 2 +- onadata/apps/api/viewsets/data_viewset.py | 2 +- onadata/apps/api/viewsets/dataview_viewset.py | 40 ++++++++++---- .../apps/api/viewsets/open_data_viewset.py | 2 +- onadata/apps/api/viewsets/osm_viewset.py | 2 +- onadata/apps/api/viewsets/team_viewset.py | 2 +- .../apps/api/viewsets/user_profile_viewset.py | 2 +- onadata/apps/api/viewsets/widget_viewset.py | 2 +- .../api/viewsets/xform_submission_viewset.py | 6 +-- onadata/apps/api/viewsets/xform_viewset.py | 2 +- .../apps/logger/management/commands/add_id.py | 4 +- .../commands/change_s3_media_permissions.py | 4 +- .../management/commands/create_backup.py | 4 +- .../commands/create_image_thumbnails.py | 12 ++--- .../management/commands/export_gps_points.py | 4 +- .../commands/export_xforms_and_instances.py | 4 +- .../commands/fix_attachments_counts.py | 6 +-- .../commands/fix_duplicate_instances.py | 4 +- .../commands/fix_submission_count.py | 4 +- .../apps/logger/management/commands/import.py | 4 +- .../management/commands/import_briefcase.py | 2 +- .../management/commands/import_forms.py | 4 +- .../management/commands/import_instances.py | 6 +-- .../management/commands/import_tools.py | 4 +- .../management/commands/move_media_to_s3.py | 4 +- .../commands/populate_osmdata_model.py | 4 +- .../logger/management/commands/publish_xls.py | 8 +-- .../commands/pull_from_aggregate.py | 2 +- .../management/commands/reapplyperms.py | 2 +- .../management/commands/restore_backup.py | 4 +- .../set_xform_surveys_with_geopoints.py | 4 +- .../commands/set_xform_surveys_with_osm.py | 4 +- .../commands/sync_deleted_instances_fix.py | 4 +- .../management/commands/update_moved_forms.py | 4 +- .../management/commands/update_xform_uuids.py | 6 +-- onadata/apps/logger/models/data_view.py | 2 +- onadata/apps/logger/models/instance.py | 2 +- .../apps/logger/models/submission_review.py | 2 +- onadata/apps/logger/models/xform.py | 14 ++--- onadata/apps/logger/views.py | 2 +- onadata/apps/logger/xform_instance_parser.py | 6 +-- onadata/apps/main/forms.py | 52 +++++++++---------- .../commands/create_enketo_express_urls.py | 4 +- .../create_metadata_for_kpi_deployed_forms.py | 4 +- .../management/commands/export_user_emails.py | 4 +- .../get_accounts_with_duplicate_id_strings.py | 4 +- .../apps/main/management/commands/mailer.py | 4 +- .../management/commands/migrate_audit_log.py | 4 +- .../management/commands/remove_odk_prefix.py | 4 +- .../commands/set_media_file_hash.py | 4 +- .../management/commands/update_enketo_urls.py | 4 +- onadata/apps/main/models/audit.py | 2 +- onadata/apps/main/models/user_profile.py | 4 +- onadata/apps/main/registration_urls.py | 12 ++--- onadata/apps/main/urls.py | 9 ++-- onadata/apps/main/views.py | 2 +- onadata/apps/messaging/constants.py | 2 +- onadata/apps/messaging/filters.py | 2 +- onadata/apps/messaging/serializers.py | 2 +- onadata/apps/messaging/urls.py | 4 +- onadata/apps/restservice/forms.py | 6 +-- .../management/commands/textit_v1_to_v2.py | 2 +- onadata/apps/restservice/models.py | 8 +-- onadata/apps/restservice/utils.py | 2 +- onadata/apps/restservice/views.py | 2 +- onadata/apps/sms_support/parser.py | 2 +- onadata/apps/sms_support/providers/smssync.py | 2 +- .../apps/sms_support/providers/telerivet.py | 2 +- onadata/apps/sms_support/providers/textit.py | 2 +- onadata/apps/sms_support/providers/twilio.py | 2 +- onadata/apps/sms_support/tools.py | 2 +- onadata/apps/sms_support/views.py | 2 +- .../apps/viewer/management/commands/import.py | 4 +- .../management/commands/import_forms.py | 4 +- .../management/commands/mark_start_times.py | 4 +- .../management/commands/set_uuid_in_xml.py | 4 +- onadata/apps/viewer/models/data_dictionary.py | 2 +- onadata/apps/viewer/models/export.py | 2 +- onadata/apps/viewer/models/parsed_instance.py | 2 +- onadata/apps/viewer/views.py | 2 +- onadata/libs/authentication.py | 12 ++--- onadata/libs/exceptions.py | 2 +- onadata/libs/models/textit_service.py | 2 +- onadata/libs/renderers/renderers.py | 4 +- .../serializers/clone_xform_serializer.py | 2 +- onadata/libs/serializers/data_serializer.py | 2 +- .../libs/serializers/dataview_serializer.py | 2 +- .../serializers/fields/organization_field.py | 2 +- .../libs/serializers/fields/project_field.py | 2 +- onadata/libs/serializers/floip_serializer.py | 2 +- .../serializers/merged_xform_serializer.py | 2 +- .../libs/serializers/metadata_serializer.py | 2 +- onadata/libs/serializers/note_serializer.py | 2 +- .../organization_member_serializer.py | 2 +- .../serializers/organization_serializer.py | 2 +- .../serializers/password_reset_serializer.py | 2 +- .../libs/serializers/project_serializer.py | 2 +- .../serializers/share_project_serializer.py | 2 +- .../share_team_project_serializer.py | 2 +- .../serializers/share_xform_serializer.py | 2 +- onadata/libs/serializers/stats_serializer.py | 2 +- .../libs/serializers/tag_list_serializer.py | 2 +- .../serializers/user_profile_serializer.py | 2 +- onadata/libs/serializers/widget_serializer.py | 2 +- onadata/libs/serializers/xform_serializer.py | 2 +- onadata/libs/tests/utils/test_middleware.py | 4 +- onadata/libs/utils/analytics.py | 6 +-- onadata/libs/utils/api_export_tools.py | 2 +- onadata/libs/utils/common_tags.py | 2 +- onadata/libs/utils/common_tools.py | 2 +- onadata/libs/utils/country_field.py | 2 +- onadata/libs/utils/csv_builder.py | 2 +- onadata/libs/utils/csv_import.py | 2 +- onadata/libs/utils/export_tools.py | 2 +- onadata/libs/utils/log.py | 2 +- onadata/libs/utils/logger_tools.py | 2 +- onadata/libs/utils/middleware.py | 4 +- onadata/libs/utils/openid_connect_tools.py | 2 +- onadata/libs/utils/quick_converter.py | 4 +- onadata/libs/utils/user_auth.py | 2 +- onadata/libs/utils/viewer_tools.py | 6 +-- 127 files changed, 277 insertions(+), 245 deletions(-) diff --git a/onadata/apps/api/admin.py b/onadata/apps/api/admin.py index a10f339dfd..1cce64406f 100644 --- a/onadata/apps/api/admin.py +++ b/onadata/apps/api/admin.py @@ -6,8 +6,11 @@ class TeamAdmin(admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" + def get_queryset(self, request): - queryset = super(TeamAdmin, self).get_queryset(request) + """Filter by request.user unless is_superuser.""" + queryset = super().get_queryset(request) if request.user.is_superuser: return queryset return queryset.filter(user=request.user) @@ -17,8 +20,11 @@ def get_queryset(self, request): class OrganizationProfileAdmin(admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" + def get_queryset(self, request): - queryset = super(OrganizationProfileAdmin, self).get_queryset(request) + """Filter by request.user unless is_superuser.""" + queryset = super().get_queryset(request) if request.user.is_superuser: return queryset return queryset.filter(user=request.user) @@ -28,8 +34,11 @@ def get_queryset(self, request): class TempTokenProfileAdmin(admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" + def get_queryset(self, request): - queryset = super(TempTokenProfileAdmin, self).get_queryset(request) + """Filter by request.user unless is_superuser.""" + queryset = super().get_queryset(request) if request.user.is_superuser: return queryset return queryset.filter(user=request.user) diff --git a/onadata/apps/api/models/odk_token.py b/onadata/apps/api/models/odk_token.py index 8ab9e6b905..2d86886f7c 100644 --- a/onadata/apps/api/models/odk_token.py +++ b/onadata/apps/api/models/odk_token.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import models from django.db.models.signals import post_save -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from cryptography.fernet import Fernet from django_digest.models import (_persist_partial_digests, diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 1092998fb7..798e5d21e0 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -21,7 +21,7 @@ from django.http import HttpResponseNotFound, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.utils.module_loading import import_string -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from guardian.shortcuts import assign_perm, get_perms, get_perms_for_model, remove_perm from kombu.exceptions import OperationalError diff --git a/onadata/apps/api/urls/v1_urls.py b/onadata/apps/api/urls/v1_urls.py index 631fb96937..15d3a830a9 100644 --- a/onadata/apps/api/urls/v1_urls.py +++ b/onadata/apps/api/urls/v1_urls.py @@ -2,7 +2,7 @@ """ Custom rest_framework Router - MultiLookupRouter. """ -from django.conf.urls import url +from django.urls import re_path from django.contrib import admin from rest_framework import routers from rest_framework.urlpatterns import format_suffix_patterns @@ -107,7 +107,7 @@ def get_urls(self): }) view = viewset.as_view(mapping, **initkwargs) name = route.name.format(basename=basename) - extra_urls.append(url(regex, view, name=name)) + extra_urls.append(re_path(regex, view, name=name)) if self.include_format_suffixes: extra_urls = format_suffix_patterns(extra_urls) diff --git a/onadata/apps/api/viewsets/attachment_viewset.py b/onadata/apps/api/viewsets/attachment_viewset.py index af38671c40..ac5af6a90f 100644 --- a/onadata/apps/api/viewsets/attachment_viewset.py +++ b/onadata/apps/api/viewsets/attachment_viewset.py @@ -1,7 +1,7 @@ from builtins import str as text from django.http import Http404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.files.storage import default_storage from django.conf import settings from rest_framework import renderers diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index 2472adcdf0..b6fa26753c 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -9,7 +9,7 @@ from django.core.files import File from django.core.validators import ValidationError from django.http import Http404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import six from rest_framework import exceptions, mixins, permissions, status, viewsets diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index d2d88bfb2f..a1c0f3817f 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -1,7 +1,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.utils import timezone from django.utils.decorators import classonlymethod -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache from rest_framework import status, viewsets from rest_framework.authtoken.models import Token diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 18196a292e..0919761ad7 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -14,7 +14,7 @@ from django.db.utils import DataError, OperationalError from django.http import Http404, StreamingHttpResponse from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import six from rest_framework import status diff --git a/onadata/apps/api/viewsets/dataview_viewset.py b/onadata/apps/api/viewsets/dataview_viewset.py index cf6a944d8f..e9d6f9da9c 100644 --- a/onadata/apps/api/viewsets/dataview_viewset.py +++ b/onadata/apps/api/viewsets/dataview_viewset.py @@ -1,5 +1,10 @@ +# -*- coding: utf-8 -*- +""" +The /dataview API endpoint implementation. +""" from django.db.models.signals import post_delete, post_save from django.http import Http404, HttpResponseBadRequest +from django.utils.translation import gettext as _ from celery.result import AsyncResult from rest_framework import status @@ -41,13 +46,18 @@ from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() def get_form_field_chart_url(url, field): - return "%s?field_name=%s" % (url, field) + """ + Returns a chart's ``url`` with the field_name ``field`` parameter appended to it. + """ + return f"{url}?field_name={field}" +# pylint: disable=too-many-ancestors class DataViewViewSet( AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet ): @@ -76,6 +86,7 @@ def get_serializer_class(self): return serializer_class + # pylint: disable=redefined-builtin,unused-argument @action(methods=["GET"], detail=True) def data(self, request, format="json", **kwargs): """Retrieve the data from the xform using this dataview""" @@ -85,6 +96,7 @@ def data(self, request, format="json", **kwargs): sort = request.GET.get("sort") query = request.GET.get("query") export_type = self.kwargs.get("format", request.GET.get("format")) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if export_type is None or export_type in ["json", "debug"]: data = DataView.query_data( @@ -102,13 +114,14 @@ def data(self, request, format="json", **kwargs): return Response(serializer.data) - else: - return custom_response_handler( - request, self.object.xform, query, export_type, dataview=self.object - ) + return custom_response_handler( + request, self.object.xform, query, export_type, dataview=self.object + ) + # pylint: disable=too-many-locals @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): + """Initiate's exports asynchronously.""" params = request.query_params job_uuid = params.get("job_uuid") export_type = params.get("format") @@ -166,8 +179,10 @@ def export_async(self, request, *args, **kwargs): data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" ) + # pylint: disable=redefined-builtin,unused-argument @action(methods=["GET"], detail=True) def form(self, request, format="json", **kwargs): + """Returns the form as either json, xml or XLS linked the dataview.""" dataview = self.get_object() xform = dataview.xform if format not in ["json", "xml", "xls"]: @@ -182,6 +197,7 @@ def form(self, request, format="json", **kwargs): @action(methods=["GET"], detail=True) def form_details(self, request, *args, **kwargs): + """Returns the dataview's form API data.""" dataview = self.get_object() xform = dataview.xform serializer = XFormSerializer(xform, context={"request": request}) @@ -190,6 +206,7 @@ def form_details(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def charts(self, request, *args, **kwargs): + """Returns the charts data for the given dataview.""" dataview = self.get_object() xform = dataview.xform serializer = self.get_serializer(dataview) @@ -215,7 +232,7 @@ def charts(self, request, *args, **kwargs): common_tags.DURATION, ] ): - raise Http404("Field %s does not not exist on the dataview" % field_name) + raise Http404(_(f"Field {field_name} does not not exist on the dataview")) if field_name or field_xpath: data = get_chart_data_for_field( @@ -245,6 +262,7 @@ def charts(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def xls_export(self, request, *args, **kwargs): + """Returns the data views XLS export files.""" dataview = self.get_object() xform = dataview.xform @@ -258,21 +276,25 @@ def xls_export(self, request, *args, **kwargs): ) def destroy(self, request, *args, **kwargs): + """Soft deletes the the dataview.""" dataview = self.get_object() user = request.user dataview.soft_delete(user) - safe_delete("{}{}".format(PROJ_OWNER_CACHE, dataview.project.pk)) + safe_delete(f"{PROJ_OWNER_CACHE}{dataview.project.pk}") return Response(status=status.HTTP_204_NO_CONTENT) +# pylint: disable=unused-argument def dataview_post_save_callback(sender, instance=None, created=False, **kwargs): - safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) + """Clear project cache post dataview save.""" + safe_delete(f"{PROJECT_LINKED_DATAVIEWS}{instance.project.pk}") def dataview_post_delete_callback(sender, instance, **kwargs): + """Clear project cache post dataview delete.""" if instance.project: - safe_delete("{}{}".format(PROJECT_LINKED_DATAVIEWS, instance.project.pk)) + safe_delete(f"{PROJECT_LINKED_DATAVIEWS}{instance.project.pk}") post_save.connect( diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index cf47a46284..c6ebcd24bc 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -7,7 +7,7 @@ from django.core.exceptions import PermissionDenied from django.http import StreamingHttpResponse from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response diff --git a/onadata/apps/api/viewsets/osm_viewset.py b/onadata/apps/api/viewsets/osm_viewset.py index 28a85b084c..243d75ae86 100644 --- a/onadata/apps/api/viewsets/osm_viewset.py +++ b/onadata/apps/api/viewsets/osm_viewset.py @@ -4,7 +4,7 @@ """ from django.http import HttpResponsePermanentRedirect from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework.exceptions import ParseError from rest_framework.permissions import AllowAny diff --git a/onadata/apps/api/viewsets/team_viewset.py b/onadata/apps/api/viewsets/team_viewset.py index aacad71502..2127746cb6 100644 --- a/onadata/apps/api/viewsets/team_viewset.py +++ b/onadata/apps/api/viewsets/team_viewset.py @@ -3,7 +3,7 @@ The /teams API endpoint implementation. """ from django.contrib.auth import get_user_model -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import status from rest_framework.decorators import action diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index b05559acf2..f02ffa851a 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -13,7 +13,7 @@ from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.utils import timezone from django.utils.module_loading import import_string -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six.moves.urllib.parse import urlencode diff --git a/onadata/apps/api/viewsets/widget_viewset.py b/onadata/apps/api/viewsets/widget_viewset.py index a82aac8a34..8c28a3bc70 100644 --- a/onadata/apps/api/viewsets/widget_viewset.py +++ b/onadata/apps/api/viewsets/widget_viewset.py @@ -1,5 +1,5 @@ from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.contrib.contenttypes.models import ContentType from rest_framework.viewsets import ModelViewSet diff --git a/onadata/apps/api/viewsets/xform_submission_viewset.py b/onadata/apps/api/viewsets/xform_submission_viewset.py index f663f604e5..c05d3df4f8 100644 --- a/onadata/apps/api/viewsets/xform_submission_viewset.py +++ b/onadata/apps/api/viewsets/xform_submission_viewset.py @@ -4,7 +4,7 @@ """ from django.conf import settings from django.http import UnreadablePostError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import mixins, permissions, status, viewsets from rest_framework.authentication import (BasicAuthentication, @@ -85,8 +85,8 @@ def get_serializer_class(self): content_type = self.request.content_type.lower() if 'application/json' in content_type: - if 'RapidProMailroom' in self.request.META.get( - 'HTTP_USER_AGENT', ''): + if 'RapidProMailroom' in self.request.headers.get( + 'User-Agent', ''): return RapidProJSONSubmissionSerializer self.request.accepted_renderer = JSONRenderer() diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index cbe33ddde3..0c17a7ad73 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -21,7 +21,7 @@ ) from django.utils import timezone from django.utils.http import urlencode -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend diff --git a/onadata/apps/logger/management/commands/add_id.py b/onadata/apps/logger/management/commands/add_id.py index f773354913..048a66ef16 100644 --- a/onadata/apps/logger/management/commands/add_id.py +++ b/onadata/apps/logger/management/commands/add_id.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand from django.conf import settings @@ -9,7 +9,7 @@ class Command(BaseCommand): args = '' - help = ugettext_lazy("Sync account with '_id'") + help = gettext_lazy("Sync account with '_id'") def handle(self, *args, **kwargs): diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 63f8f23b1c..7ca4be7068 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -4,11 +4,11 @@ from django.core.management.base import BaseCommand, CommandError from django.core.files.storage import get_storage_class -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Makes all s3 files private") + help = gettext_lazy("Makes all s3 files private") def handle(self, *args, **kwargs): permissions = ('private', 'public-read', 'authenticated-read') diff --git a/onadata/apps/logger/management/commands/create_backup.py b/onadata/apps/logger/management/commands/create_backup.py index 6e95bd3bf0..29c91941a7 100644 --- a/onadata/apps/logger/management/commands/create_backup.py +++ b/onadata/apps/logger/management/commands/create_backup.py @@ -1,7 +1,7 @@ import os from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.translation import gettext_lazy, gettext as _ from django.contrib.auth.models import User from onadata.apps.logger.models import XForm @@ -10,7 +10,7 @@ class Command(BaseCommand): args = "outfile username [id_string]" - help = ugettext_lazy( + help = gettext_lazy( "Create a zip backup of a form and all its submissions") def handle(self, *args, **options): diff --git a/onadata/apps/logger/management/commands/create_image_thumbnails.py b/onadata/apps/logger/management/commands/create_image_thumbnails.py index a03718f33b..d0b83865a5 100644 --- a/onadata/apps/logger/management/commands/create_image_thumbnails.py +++ b/onadata/apps/logger/management/commands/create_image_thumbnails.py @@ -3,8 +3,8 @@ from django.contrib.auth.models import User from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.xform import XForm @@ -16,21 +16,21 @@ class Command(BaseCommand): - help = ugettext_lazy("Creates thumbnails for " + help = gettext_lazy("Creates thumbnails for " "all form images and stores them") def add_arguments(self, parser): parser.add_argument( '-u', '--username', - help=ugettext_lazy("Username of the form user")) + help=gettext_lazy("Username of the form user")) parser.add_argument( - '-i', '--id_string', help=ugettext_lazy("id string of the form")) + '-i', '--id_string', help=gettext_lazy("id string of the form")) parser.add_argument( '-f', '--force', action='store_false', - help=ugettext_lazy("regenerate thumbnails if they exist.")) + help=gettext_lazy("regenerate thumbnails if they exist.")) def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( diff --git a/onadata/apps/logger/management/commands/export_gps_points.py b/onadata/apps/logger/management/commands/export_gps_points.py index 15f5848b77..9e32605fa4 100644 --- a/onadata/apps/logger/management/commands/export_gps_points.py +++ b/onadata/apps/logger/management/commands/export_gps_points.py @@ -2,14 +2,14 @@ import csv from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Export all gps points with their timestamps") + help = gettext_lazy("Export all gps points with their timestamps") def handle(self, *args, **kwargs): with open('gps_points_export.csv', 'w') as csvfile: diff --git a/onadata/apps/logger/management/commands/export_xforms_and_instances.py b/onadata/apps/logger/management/commands/export_xforms_and_instances.py index d983897cdb..a6c714f160 100644 --- a/onadata/apps/logger/management/commands/export_xforms_and_instances.py +++ b/onadata/apps/logger/management/commands/export_xforms_and_instances.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.core.serializers import serialize -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.conf import settings from onadata.apps.logger.models import XForm, Instance @@ -12,7 +12,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Export ODK forms and instances to JSON.") + help = gettext_lazy("Export ODK forms and instances to JSON.") def handle(self, *args, **kwargs): fixtures_dir = os.path.join(PROJECT_ROOT, "json_xform_fixtures") diff --git a/onadata/apps/logger/management/commands/fix_attachments_counts.py b/onadata/apps/logger/management/commands/fix_attachments_counts.py index 00608cb913..84c008f9b3 100644 --- a/onadata/apps/logger/management/commands/fix_attachments_counts.py +++ b/onadata/apps/logger/management/commands/fix_attachments_counts.py @@ -6,8 +6,8 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from multidb.pinning import use_master from onadata.apps.logger.models.attachment import get_original_filename @@ -32,7 +32,7 @@ class Command(BaseCommand): Fix attachments count command. """ args = 'username' - help = ugettext_lazy("Fix attachments count.") + help = gettext_lazy("Fix attachments count.") def add_arguments(self, parser): parser.add_argument('username') diff --git a/onadata/apps/logger/management/commands/fix_duplicate_instances.py b/onadata/apps/logger/management/commands/fix_duplicate_instances.py index 618dfba306..fe79ca0515 100644 --- a/onadata/apps/logger/management/commands/fix_duplicate_instances.py +++ b/onadata/apps/logger/management/commands/fix_duplicate_instances.py @@ -3,13 +3,13 @@ from django.core.management.base import BaseCommand from django.db import connection -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance class Command(BaseCommand): - help = ugettext_lazy("Fix duplicate instances by merging the attachments.") + help = gettext_lazy("Fix duplicate instances by merging the attachments.") def query_data(self, sql): cursor = connection.cursor() diff --git a/onadata/apps/logger/management/commands/fix_submission_count.py b/onadata/apps/logger/management/commands/fix_submission_count.py index 1f920239a9..06e705d03d 100644 --- a/onadata/apps/logger/management/commands/fix_submission_count.py +++ b/onadata/apps/logger/management/commands/fix_submission_count.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from django.db import transaction -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance from onadata.apps.logger.models import XForm @@ -11,7 +11,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Fix num of submissions") + help = gettext_lazy("Fix num of submissions") def handle(self, *args, **kwargs): i = 0 diff --git a/onadata/apps/logger/management/commands/import.py b/onadata/apps/logger/management/commands/import.py index 7c6dc3a25d..fe1894423c 100644 --- a/onadata/apps/logger/management/commands/import.py +++ b/onadata/apps/logger/management/commands/import.py @@ -4,11 +4,11 @@ import os from django.core.management.base import BaseCommand from django.core.management import call_command -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): path = args[0] diff --git a/onadata/apps/logger/management/commands/import_briefcase.py b/onadata/apps/logger/management/commands/import_briefcase.py index 57ba797d8f..e203504fae 100644 --- a/onadata/apps/logger/management/commands/import_briefcase.py +++ b/onadata/apps/logger/management/commands/import_briefcase.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.briefcase_client import BriefcaseClient diff --git a/onadata/apps/logger/management/commands/import_forms.py b/onadata/apps/logger/management/commands/import_forms.py index 7c0c0f59a7..6fd8d72d62 100644 --- a/onadata/apps/logger/management/commands/import_forms.py +++ b/onadata/apps/logger/management/commands/import_forms.py @@ -6,13 +6,13 @@ import os from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") + help = gettext_lazy("Import a folder of XForms for ODK.") def handle(self, *args, **kwargs): path = args[0] diff --git a/onadata/apps/logger/management/commands/import_instances.py b/onadata/apps/logger/management/commands/import_instances.py index ec61c47b78..b54a364025 100644 --- a/onadata/apps/logger/management/commands/import_instances.py +++ b/onadata/apps/logger/management/commands/import_instances.py @@ -9,8 +9,8 @@ from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.logger.import_tools import ( import_instances_from_path, @@ -27,7 +27,7 @@ class Command(BaseCommand): """ args = "username path" - help = ugettext_lazy( + help = gettext_lazy( "Import a zip file, a directory containing zip files " "or a directory of ODK instances" ) diff --git a/onadata/apps/logger/management/commands/import_tools.py b/onadata/apps/logger/management/commands/import_tools.py index 9383e143ef..399ae0b609 100644 --- a/onadata/apps/logger/management/commands/import_tools.py +++ b/onadata/apps/logger/management/commands/import_tools.py @@ -7,7 +7,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from onadata.libs.logger.import_tools import import_instances_from_zip from onadata.libs.logger.models import Instance @@ -17,7 +17,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): if args.__len__() < 2: diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index f2d8fbc213..ff5d3772ab 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -3,7 +3,7 @@ from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.attachment import upload_to as\ @@ -13,7 +13,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Moves all attachments and xls files " + help = gettext_lazy("Moves all attachments and xls files " "to s3 from the local file system storage.") def handle(self, *args, **kwargs): diff --git a/onadata/apps/logger/management/commands/populate_osmdata_model.py b/onadata/apps/logger/management/commands/populate_osmdata_model.py index 58603cc276..9125e6b3a2 100644 --- a/onadata/apps/logger/management/commands/populate_osmdata_model.py +++ b/onadata/apps/logger/management/commands/populate_osmdata_model.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand from onadata.apps.logger.models import Attachment @@ -9,7 +9,7 @@ class Command(BaseCommand): args = '' - help = ugettext_lazy("Populate OsmData model with osm info.") + help = gettext_lazy("Populate OsmData model with osm info.") def handle(self, *args, **kwargs): xforms = XForm.objects.filter(instances_with_osm=True) diff --git a/onadata/apps/logger/management/commands/publish_xls.py b/onadata/apps/logger/management/commands/publish_xls.py index d04a13e3a6..34d323e359 100644 --- a/onadata/apps/logger/management/commands/publish_xls.py +++ b/onadata/apps/logger/management/commands/publish_xls.py @@ -2,8 +2,8 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from pyxform.builder import create_survey_from_xls from onadata.apps.logger.models.project import Project @@ -15,7 +15,7 @@ class Command(BaseCommand): args = 'xls_file username project' - help = ugettext_lazy("Publish an XLS file with the option of replacing an" + help = gettext_lazy("Publish an XLS file with the option of replacing an" "existing one") def add_arguments(self, parser): @@ -28,7 +28,7 @@ def add_arguments(self, parser): '--replace', action='store_true', dest='replace', - help=ugettext_lazy("Replace existing form if any")) + help=gettext_lazy("Replace existing form if any")) def handle(self, *args, **options): try: diff --git a/onadata/apps/logger/management/commands/pull_from_aggregate.py b/onadata/apps/logger/management/commands/pull_from_aggregate.py index cbb0bdaea5..d1dfb1a2ad 100644 --- a/onadata/apps/logger/management/commands/pull_from_aggregate.py +++ b/onadata/apps/logger/management/commands/pull_from_aggregate.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.briefcase_client import BriefcaseClient diff --git a/onadata/apps/logger/management/commands/reapplyperms.py b/onadata/apps/logger/management/commands/reapplyperms.py index 20060f81f6..1dcc9feebb 100644 --- a/onadata/apps/logger/management/commands/reapplyperms.py +++ b/onadata/apps/logger/management/commands/reapplyperms.py @@ -7,7 +7,7 @@ from onadata.apps.logger.models import XForm from django.db.models import Q from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from multidb.pinning import use_master diff --git a/onadata/apps/logger/management/commands/restore_backup.py b/onadata/apps/logger/management/commands/restore_backup.py index 72d2552f18..5e9484f908 100644 --- a/onadata/apps/logger/management/commands/restore_backup.py +++ b/onadata/apps/logger/management/commands/restore_backup.py @@ -3,14 +3,14 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.translation import gettext_lazy, gettext as _ from onadata.libs.utils.backup_tools import restore_backup_from_zip class Command(BaseCommand): args = 'username input_file' - help = ugettext_lazy("Restore a zip backup of a form and all its" + help = gettext_lazy("Restore a zip backup of a form and all its" " submissions") def handle(self, *args, **options): diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py index dbb106045d..b40780779a 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py @@ -2,14 +2,14 @@ # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") + help = gettext_lazy("Import a folder of XForms for ODK.") def handle(self, *args, **kwargs): xforms = XForm.objects.all() diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py index 79b5664c87..f6e3445006 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py @@ -2,7 +2,7 @@ # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.xform import XForm @@ -10,7 +10,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Set xform.instances_with_osm") + help = gettext_lazy("Set xform.instances_with_osm") def handle(self, *args, **kwargs): pks = Attachment.objects.filter( diff --git a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py index a53dff842a..d515edc7ca 100644 --- a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py +++ b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py @@ -6,13 +6,13 @@ from django.core.management import BaseCommand from django.utils import timezone from django.utils.dateparse import parse_datetime -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import Instance class Command(BaseCommand): - help = ugettext_lazy("Fixes deleted instances by syncing " + help = gettext_lazy("Fixes deleted instances by syncing " "deleted items from mongo.") def handle(self, *args, **kwargs): diff --git a/onadata/apps/logger/management/commands/update_moved_forms.py b/onadata/apps/logger/management/commands/update_moved_forms.py index 9e10774134..6b7749fbe4 100644 --- a/onadata/apps/logger/management/commands/update_moved_forms.py +++ b/onadata/apps/logger/management/commands/update_moved_forms.py @@ -1,12 +1,12 @@ from django.core.management import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.project import Project from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Ensures all the forms are owned by the project" + help = gettext_lazy("Ensures all the forms are owned by the project" " owner") def handle(self, *args, **kwargs): diff --git a/onadata/apps/logger/management/commands/update_xform_uuids.py b/onadata/apps/logger/management/commands/update_xform_uuids.py index d6b7c2db9a..ff08e0428b 100644 --- a/onadata/apps/logger/management/commands/update_xform_uuids.py +++ b/onadata/apps/logger/management/commands/update_xform_uuids.py @@ -4,20 +4,20 @@ import csv from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.xform import (DuplicateUUIDError, XForm, update_xform_uuid) class Command(BaseCommand): - help = ugettext_lazy( + help = gettext_lazy( "Use a csv file with username, id_string and new_uuid to set new" " uuids") def add_arguments(self, parser): parser.add_argument( - '-f', '--file', help=ugettext_lazy("Path to csv file")) + '-f', '--file', help=gettext_lazy("Path to csv file")) def handle(self, *args, **kwargs): # all options are required diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 3f187f345d..21a8525c68 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -10,7 +10,7 @@ from django.db import connection from django.db.models.signals import post_delete, post_save from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.db.utils import DataError from onadata.apps.viewer.parsed_instance_tools import get_where_clause diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index bd59e70c0e..dc29cb6566 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -15,7 +15,7 @@ from django.db.models.signals import post_delete, post_save from django.urls import reverse from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import pytz from deprecated import deprecated diff --git a/onadata/apps/logger/models/submission_review.py b/onadata/apps/logger/models/submission_review.py index 34aac1a413..82ea73b94d 100644 --- a/onadata/apps/logger/models/submission_review.py +++ b/onadata/apps/logger/models/submission_review.py @@ -8,7 +8,7 @@ from django.db import models from django.db.models.signals import post_save from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def update_instance_json_on_save(sender, instance, **kwargs): diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 32b663c75b..248008cb34 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -22,8 +22,8 @@ from django.urls import reverse from django.utils import timezone from django.utils.html import conditional_escape -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from six import iteritems from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict @@ -830,12 +830,12 @@ class XForm(XFormMixin, BaseModel): # the following fields are filled in automatically sms_id_string = models.SlugField( editable=False, - verbose_name=ugettext_lazy("SMS ID"), + verbose_name=gettext_lazy("SMS ID"), max_length=MAX_ID_LENGTH, default="", ) id_string = models.SlugField( - editable=False, verbose_name=ugettext_lazy("ID"), max_length=MAX_ID_LENGTH + editable=False, verbose_name=gettext_lazy("ID"), max_length=MAX_ID_LENGTH ) title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH) date_created = models.DateTimeField(auto_now_add=True) @@ -880,8 +880,8 @@ class Meta: ("user", "id_string", "project"), ("user", "sms_id_string", "project"), ) - verbose_name = ugettext_lazy("XForm") - verbose_name_plural = ugettext_lazy("XForms") + verbose_name = gettext_lazy("XForm") + verbose_name_plural = gettext_lazy("XForms") ordering = ("pk",) permissions = ( ("view_xform_all", _("Can view all associated data")), @@ -1121,7 +1121,7 @@ def submission_count(self, force_update=False): return self.num_of_submissions - submission_count.short_description = ugettext_lazy("Submission Count") + submission_count.short_description = gettext_lazy("Submission Count") @property def submission_count_for_today(self): diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 11b3a9e849..dca577b202 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -26,7 +26,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_http_methods, require_POST from django_digest import HttpDigestAuthenticator diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index cb4cdcae48..b12a2319d4 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -8,8 +8,8 @@ import dateutil.parser -from django.utils.encoding import smart_text, smart_str -from django.utils.translation import ugettext as _ +from django.utils.encoding import smart_str +from django.utils.translation import gettext as _ from onadata.libs.utils.common_tags import XFORM_ID_STRING, VERSION @@ -168,7 +168,7 @@ def clean_and_parse_xml(xml_string): Returns an XML object via minidom.parseString(xml_string) """ clean_xml_str = xml_string.strip() - clean_xml_str = re.sub(r">\s+<", "><", smart_text(clean_xml_str)) + clean_xml_str = re.sub(r">\s+<", "><", smart_str(clean_xml_str)) xml_obj = minidom.parseString(smart_str(clean_xml_str)) return xml_obj diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index 9797d24731..7a4afd10ea 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -15,8 +15,8 @@ from django.core.files.storage import default_storage from django.core.validators import URLValidator from django.forms import ModelForm -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from registration.forms import RegistrationFormUniqueEmail # pylint: disable=ungrouped-imports @@ -28,29 +28,29 @@ from onadata.libs.utils.user_auth import get_user_default_project FORM_LICENSES_CHOICES = ( - ("No License", ugettext_lazy("No License")), + ("No License", gettext_lazy("No License")), ( "https://creativecommons.org/licenses/by/3.0/", - ugettext_lazy("Attribution CC BY"), + gettext_lazy("Attribution CC BY"), ), ( "https://creativecommons.org/licenses/by-sa/3.0/", - ugettext_lazy("Attribution-ShareAlike CC BY-SA"), + gettext_lazy("Attribution-ShareAlike CC BY-SA"), ), ) DATA_LICENSES_CHOICES = ( - ("No License", ugettext_lazy("No License")), - ("http://opendatacommons.org/licenses/pddl/summary/", ugettext_lazy("PDDL")), - ("http://opendatacommons.org/licenses/by/summary/", ugettext_lazy("ODC-BY")), - ("http://opendatacommons.org/licenses/odbl/summary/", ugettext_lazy("ODBL")), + ("No License", gettext_lazy("No License")), + ("http://opendatacommons.org/licenses/pddl/summary/", gettext_lazy("PDDL")), + ("http://opendatacommons.org/licenses/by/summary/", gettext_lazy("ODC-BY")), + ("http://opendatacommons.org/licenses/odbl/summary/", gettext_lazy("ODBL")), ) PERM_CHOICES = ( - ("view", ugettext_lazy("Can view")), - ("edit", ugettext_lazy("Can edit")), - ("report", ugettext_lazy("Can submit to")), - ("remove", ugettext_lazy("Remove permissions")), + ("view", gettext_lazy("Can view")), + ("edit", gettext_lazy("Can edit")), + ("report", gettext_lazy("Can submit to")), + ("remove", gettext_lazy("Remove permissions")), ) VALID_XLSFORM_CONTENT_TYPES = [ @@ -231,7 +231,7 @@ class SourceForm(forms.Form): Source document form. """ - source = forms.FileField(label=ugettext_lazy("Source document"), required=True) + source = forms.FileField(label=gettext_lazy("Source document"), required=True) class SupportDocForm(forms.Form): @@ -239,7 +239,7 @@ class SupportDocForm(forms.Form): Supporting document. """ - doc = forms.FileField(label=ugettext_lazy("Supporting document"), required=True) + doc = forms.FileField(label=gettext_lazy("Supporting document"), required=True) class MediaForm(forms.Form): @@ -247,7 +247,7 @@ class MediaForm(forms.Form): Media file upload form. """ - media = forms.FileField(label=ugettext_lazy("Media upload"), required=True) + media = forms.FileField(label=gettext_lazy("Media upload"), required=True) def clean_media(self): """ @@ -270,7 +270,7 @@ class MapboxLayerForm(forms.Form): attribution = forms.CharField( widget=forms.TextInput(), required=False, max_length=255 ) - link = forms.URLField(label=ugettext_lazy("JSONP url"), required=True) + link = forms.URLField(label=gettext_lazy("JSONP url"), required=True) class QuickConverterFile(forms.Form): @@ -278,7 +278,7 @@ class QuickConverterFile(forms.Form): Uploads XLSForm form. """ - xls_file = forms.FileField(label=ugettext_lazy("XLS File"), required=False) + xls_file = forms.FileField(label=gettext_lazy("XLS File"), required=False) class QuickConverterURL(forms.Form): @@ -286,7 +286,7 @@ class QuickConverterURL(forms.Form): Uploads XLSForm from a URL. """ - xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) + xls_url = forms.URLField(label=gettext_lazy("XLS URL"), required=False) class QuickConverterDropboxURL(forms.Form): @@ -294,7 +294,7 @@ class QuickConverterDropboxURL(forms.Form): Uploads XLSForm from Dropbox. """ - dropbox_xls_url = forms.URLField(label=ugettext_lazy("XLS URL"), required=False) + dropbox_xls_url = forms.URLField(label=gettext_lazy("XLS URL"), required=False) class QuickConverterCsvFile(forms.Form): @@ -302,7 +302,7 @@ class QuickConverterCsvFile(forms.Form): Uploads CSV XLSForm. """ - csv_url = forms.URLField(label=ugettext_lazy("CSV URL"), required=False) + csv_url = forms.URLField(label=gettext_lazy("CSV URL"), required=False) class QuickConverterTextXlsForm(forms.Form): @@ -311,7 +311,7 @@ class QuickConverterTextXlsForm(forms.Form): """ text_xls_form = forms.CharField( - label=ugettext_lazy("XLSForm Representation"), required=False + label=gettext_lazy("XLSForm Representation"), required=False ) @@ -320,7 +320,7 @@ class QuickConverterXmlFile(forms.Form): Uploads an XForm XML. """ - xml_file = forms.FileField(label=ugettext_lazy("XML File"), required=False) + xml_file = forms.FileField(label=gettext_lazy("XML File"), required=False) class QuickConverterFloipFile(forms.Form): @@ -329,7 +329,7 @@ class QuickConverterFloipFile(forms.Form): """ floip_file = forms.FileField( - label=ugettext_lazy("FlOIP results data packages descriptor File"), + label=gettext_lazy("FlOIP results data packages descriptor File"), required=False, ) @@ -461,10 +461,10 @@ class ActivateSMSSupportForm(forms.Form): coerce=lambda x: x == "True", choices=((False, "No"), (True, "Yes")), widget=forms.Select, - label=ugettext_lazy("Enable SMS Support"), + label=gettext_lazy("Enable SMS Support"), ) sms_id_string = forms.CharField( - max_length=50, required=True, label=ugettext_lazy("SMS Keyword") + max_length=50, required=True, label=gettext_lazy("SMS Keyword") ) def clean_sms_id_string(self): diff --git a/onadata/apps/main/management/commands/create_enketo_express_urls.py b/onadata/apps/main/management/commands/create_enketo_express_urls.py index a926566901..c9658fbd0d 100644 --- a/onadata/apps/main/management/commands/create_enketo_express_urls.py +++ b/onadata/apps/main/management/commands/create_enketo_express_urls.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand, CommandError from django.http import HttpRequest -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm from onadata.libs.utils.model_tools import queryset_iterator @@ -9,7 +9,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Create enketo url including preview") + help = gettext_lazy("Create enketo url including preview") def add_arguments(self, parser): parser.add_argument( diff --git a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py index 9cea2a0914..c5e8105a29 100644 --- a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py +++ b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.db import connection from onadata.apps.main.models import MetaData @@ -9,7 +9,7 @@ class Command(BaseCommand): - help = ugettext_lazy("Create metadata for kpi forms that are not editable") + help = gettext_lazy("Create metadata for kpi forms that are not editable") def handle(self, *args, **kwargs): cursor = connection.cursor() diff --git a/onadata/apps/main/management/commands/export_user_emails.py b/onadata/apps/main/management/commands/export_user_emails.py index 4be652abe1..42289e9067 100644 --- a/onadata/apps/main/management/commands/export_user_emails.py +++ b/onadata/apps/main/management/commands/export_user_emails.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models import UserProfile from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Export users and emails") + help = gettext_lazy("Export users and emails") def handle(self, *args, **kwargs): self.stdout.write( diff --git a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py index 657e34a83e..3f261f3137 100644 --- a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py +++ b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py @@ -1,12 +1,12 @@ from django.core.management.base import BaseCommand from onadata.apps.logger.models.xform import XForm -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from django.db.models import Count from pprint import pprint class Command(BaseCommand): - help = ugettext_lazy("Retrieves accounts with duplicate id_strings") + help = gettext_lazy("Retrieves accounts with duplicate id_strings") def handle(self, *args, **kwargs): duplicates = XForm.objects.values( diff --git a/onadata/apps/main/management/commands/mailer.py b/onadata/apps/main/management/commands/mailer.py index af8406dbb2..014c07d1e3 100644 --- a/onadata/apps/main/management/commands/mailer.py +++ b/onadata/apps/main/management/commands/mailer.py @@ -1,13 +1,13 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User from django.template.loader import get_template -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from templated_email import send_templated_mail class Command(BaseCommand): - help = ugettext_lazy("Send an email to all onadata users") + help = gettext_lazy("Send an email to all onadata users") def add_arguments(self, parser): parser.add_argument("-m", "--message", dest="message", default=False) diff --git a/onadata/apps/main/management/commands/migrate_audit_log.py b/onadata/apps/main/management/commands/migrate_audit_log.py index ceacf5c51d..4abfd380eb 100644 --- a/onadata/apps/main/management/commands/migrate_audit_log.py +++ b/onadata/apps/main/management/commands/migrate_audit_log.py @@ -1,12 +1,12 @@ from django.conf import settings from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models.audit import Audit class Command(BaseCommand): - help = ugettext_lazy("migrate audit log from mongo to postgres") + help = gettext_lazy("migrate audit log from mongo to postgres") def handle(self, *args, **kwargs): auditlog = settings.MONGO_DB.auditlog diff --git a/onadata/apps/main/management/commands/remove_odk_prefix.py b/onadata/apps/main/management/commands/remove_odk_prefix.py index 9d17e8b35e..093b51bff4 100644 --- a/onadata/apps/main/management/commands/remove_odk_prefix.py +++ b/onadata/apps/main/management/commands/remove_odk_prefix.py @@ -1,10 +1,10 @@ from django.core.management.base import BaseCommand from django.db import connection -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Remove from logger and viewer apps") + help = gettext_lazy("Remove from logger and viewer apps") option_list = BaseCommand.option_list diff --git a/onadata/apps/main/management/commands/set_media_file_hash.py b/onadata/apps/main/management/commands/set_media_file_hash.py index 44f12fba06..b9fa0535e9 100644 --- a/onadata/apps/main/management/commands/set_media_file_hash.py +++ b/onadata/apps/main/management/commands/set_media_file_hash.py @@ -1,12 +1,12 @@ from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models import MetaData from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Set media file_hash for all existing media files") + help = gettext_lazy("Set media file_hash for all existing media files") option_list = BaseCommand.option_list diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 1611a1e722..96a66d06fd 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -5,7 +5,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import Q from django.http import HttpRequest -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.main.models.meta_data import MetaData from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url @@ -14,7 +14,7 @@ class Command(BaseCommand): """Updates enketo preview urls in MetaData model""" - help = ugettext_lazy("Updates enketo preview urls in MetaData model") + help = gettext_lazy("Updates enketo preview urls in MetaData model") def add_arguments(self, parser): parser.add_argument( diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 3e11f55834..8443b67e7a 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -7,7 +7,7 @@ from django.db import models from django.db import connection -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ DEFAULT_LIMIT = 1000 diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 755390b8a8..c3cc98528e 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -6,7 +6,7 @@ from django.contrib.auth import get_user_model from django.db import models from django.db.models.signals import post_save, pre_save -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy import requests from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase @@ -44,7 +44,7 @@ class UserProfile(models.Model): twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( - default=False, verbose_name=ugettext_lazy("Require Phone Authentication") + default=False, verbose_name=gettext_lazy("Require Phone Authentication") ) address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) diff --git a/onadata/apps/main/registration_urls.py b/onadata/apps/main/registration_urls.py index 7038f19306..42af15af0a 100644 --- a/onadata/apps/main/registration_urls.py +++ b/onadata/apps/main/registration_urls.py @@ -8,7 +8,7 @@ """ -from django.conf.urls import url, include +from django.urls import include, path, re_path from django.views.generic import TemplateView from registration.backends.default.views import ActivationView @@ -16,7 +16,7 @@ from onadata.apps.main.forms import RegistrationFormUserProfile urlpatterns = [ - url(r'^activate/complete/$', + path('activate/complete/', TemplateView.as_view( template_name='registration/activation_complete.html'), name='registration_activation_complete'), @@ -24,15 +24,15 @@ # [a-fA-F0-9]{40} because a bad activation key should still get to the view # that way it can return a sensible "invalid key" message instead of a # confusing 404. - url(r'^activate/(?P\w+)/$', + re_path(r'^activate/(?P\w+)/$', ActivationView.as_view(), name='registration_activate'), - url(r'^register/$', + path('register/', FHRegistrationView.as_view(form_class=RegistrationFormUserProfile), name='registration_register'), - url(r'^register/complete/$', + path('register/complete/', TemplateView.as_view( template_name='registration/registration_complete.html'), name='registration_complete'), - url(r'', include('registration.auth_urls')), + re_path(r'', include('registration.auth_urls')), ] diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index 8bdd6bfe6d..6f95fe300c 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -3,7 +3,8 @@ import django from django.conf import settings -from django.conf.urls import include, url, i18n +from django.urls import include, re_path +from django.conf.urls import i18n from django.contrib.staticfiles import views as staticfiles_views from django.urls import re_path from django.views.generic import RedirectView @@ -38,10 +39,10 @@ urlpatterns = [ # change Language re_path(r'^i18n/', include(i18n)), - url('^api/v1/', include(api_v1_router.urls)), - url('^api/v2/', include(api_v2_router.urls)), + re_path('^api/v1/', include(api_v1_router.urls)), + re_path('^api/v2/', include(api_v2_router.urls)), # open id connect urls - url(r"^", include("oidc.urls")), + re_path(r"^", include("oidc.urls")), re_path(r'^api-docs/', RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), re_path(r'^api/$', diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index ce79411e1a..3d2710ed86 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -28,7 +28,7 @@ from django.shortcuts import get_object_or_404, render from django.template import loader from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.utils.html import conditional_escape from django.views.decorators.http import require_GET, require_http_methods, require_POST from guardian.shortcuts import assign_perm, remove_perm diff --git a/onadata/apps/messaging/constants.py b/onadata/apps/messaging/constants.py index be5e1f04e2..dfe7599ddf 100644 --- a/onadata/apps/messaging/constants.py +++ b/onadata/apps/messaging/constants.py @@ -7,7 +7,7 @@ from builtins import str as text -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ XFORM = text('xform') PROJECT = text('project') diff --git a/onadata/apps/messaging/filters.py b/onadata/apps/messaging/filters.py index 28e730d3bc..8250e2d703 100644 --- a/onadata/apps/messaging/filters.py +++ b/onadata/apps/messaging/filters.py @@ -6,7 +6,7 @@ from actstream.models import Action from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, filters from django_filters import rest_framework as rest_filters diff --git a/onadata/apps/messaging/serializers.py b/onadata/apps/messaging/serializers.py index c74866b936..ca71d4942e 100644 --- a/onadata/apps/messaging/serializers.py +++ b/onadata/apps/messaging/serializers.py @@ -14,7 +14,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.http import HttpRequest -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers from onadata.apps.messaging.constants import MESSAGE, MESSAGE_VERBS diff --git a/onadata/apps/messaging/urls.py b/onadata/apps/messaging/urls.py index 2c8f45acc3..2fe9163fd8 100644 --- a/onadata/apps/messaging/urls.py +++ b/onadata/apps/messaging/urls.py @@ -2,7 +2,7 @@ """ Messaging urls module. """ -from django.conf.urls import include, url +from django.urls import include, re_path from rest_framework import routers from onadata.apps.messaging.viewsets import MessagingViewSet @@ -11,5 +11,5 @@ router.register(r'messaging', MessagingViewSet) urlpatterns = [ # pylint: disable=C0103 - url(r'^api/v1/', include(router.urls)), + re_path(r'^api/v1/', include(router.urls)), ] diff --git a/onadata/apps/restservice/forms.py b/onadata/apps/restservice/forms.py index 52ff46471d..2f6d052fa8 100644 --- a/onadata/apps/restservice/forms.py +++ b/onadata/apps/restservice/forms.py @@ -1,11 +1,11 @@ from django import forms -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.restservice import SERVICE_CHOICES class RestServiceForm(forms.Form): service_name = \ - forms.CharField(max_length=50, label=ugettext_lazy(u"Service Name"), + forms.CharField(max_length=50, label=gettext_lazy(u"Service Name"), widget=forms.Select(choices=SERVICE_CHOICES)) - service_url = forms.URLField(label=ugettext_lazy(u"Service URL")) + service_url = forms.URLField(label=gettext_lazy(u"Service URL")) diff --git a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py index e3fcb79f09..4248598302 100644 --- a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py +++ b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py @@ -3,7 +3,7 @@ import re from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.restservice.models import RestService from onadata.libs.utils.common_tags import TEXTIT diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 2632a70afc..73c27ab23d 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -7,7 +7,7 @@ from django.conf import settings from django.db import models from django.db.models.signals import post_delete, post_save -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models.xform import XForm from onadata.apps.main.models import MetaData @@ -24,16 +24,16 @@ class Meta: app_label = "restservice" unique_together = ("service_url", "xform", "name") - service_url = models.URLField(ugettext_lazy("Service URL")) + service_url = models.URLField(gettext_lazy("Service URL")) xform = models.ForeignKey(XForm, on_delete=models.CASCADE) name = models.CharField(max_length=50, choices=SERVICE_CHOICES) date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True) date_modified = models.DateTimeField(auto_now=True, null=True, blank=True) active = models.BooleanField( - ugettext_lazy("Active"), default=True, blank=False, null=False + gettext_lazy("Active"), default=True, blank=False, null=False ) inactive_reason = models.TextField( - ugettext_lazy("Inactive reason"), blank=True, default="" + gettext_lazy("Inactive reason"), blank=True, default="" ) def __str__(self): diff --git a/onadata/apps/restservice/utils.py b/onadata/apps/restservice/utils.py index a16e7061f5..ff1c9caf68 100644 --- a/onadata/apps/restservice/utils.py +++ b/onadata/apps/restservice/utils.py @@ -1,6 +1,6 @@ import logging -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.restservice.models import RestService from onadata.libs.utils.common_tags import GOOGLE_SHEET diff --git a/onadata/apps/restservice/views.py b/onadata/apps/restservice/views.py index 4878a488f0..194935e8f0 100644 --- a/onadata/apps/restservice/views.py +++ b/onadata/apps/restservice/views.py @@ -7,7 +7,7 @@ from django.template.base import Template from django.template.context import Context from django.template.loader import render_to_string -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.viewer_tools import get_form from onadata.apps.restservice.forms import RestServiceForm diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index a6574478a1..4cdd923fb3 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -6,7 +6,7 @@ from datetime import datetime, date from io import BytesIO -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_PARSING_ERROR,\ diff --git a/onadata/apps/sms_support/providers/smssync.py b/onadata/apps/sms_support/providers/smssync.py index e5159bc000..b7bd71a781 100644 --- a/onadata/apps/sms_support/providers/smssync.py +++ b/onadata/apps/sms_support/providers/smssync.py @@ -12,7 +12,7 @@ from django.http import HttpResponse from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt diff --git a/onadata/apps/sms_support/providers/telerivet.py b/onadata/apps/sms_support/providers/telerivet.py index ae9f37987a..5bf25030f0 100644 --- a/onadata/apps/sms_support/providers/telerivet.py +++ b/onadata/apps/sms_support/providers/telerivet.py @@ -9,7 +9,7 @@ from django.http import HttpResponse from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt diff --git a/onadata/apps/sms_support/providers/textit.py b/onadata/apps/sms_support/providers/textit.py index 03fa01ac1a..21b69efa55 100644 --- a/onadata/apps/sms_support/providers/textit.py +++ b/onadata/apps/sms_support/providers/textit.py @@ -13,7 +13,7 @@ from django.http import HttpResponse from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt diff --git a/onadata/apps/sms_support/providers/twilio.py b/onadata/apps/sms_support/providers/twilio.py index c9fd62e815..79bde0b75e 100644 --- a/onadata/apps/sms_support/providers/twilio.py +++ b/onadata/apps/sms_support/providers/twilio.py @@ -14,7 +14,7 @@ from dict2xml import dict2xml from django.http import HttpResponse from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index cd3c1a58d5..d0690c31aa 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -10,7 +10,7 @@ from django.contrib.auth.models import User from django.core.files.uploadedfile import InMemoryUploadedFile from django.http import HttpRequest -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm from onadata.apps.logger.models.instance import FormInactiveError diff --git a/onadata/apps/sms_support/views.py b/onadata/apps/sms_support/views.py index e28f2b410e..f8c1a82eb3 100644 --- a/onadata/apps/sms_support/views.py +++ b/onadata/apps/sms_support/views.py @@ -6,7 +6,7 @@ import json from django.http import HttpResponse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_POST diff --git a/onadata/apps/viewer/management/commands/import.py b/onadata/apps/viewer/management/commands/import.py index 7c6dc3a25d..fe1894423c 100644 --- a/onadata/apps/viewer/management/commands/import.py +++ b/onadata/apps/viewer/management/commands/import.py @@ -4,11 +4,11 @@ import os from django.core.management.base import BaseCommand from django.core.management import call_command -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy class Command(BaseCommand): - help = ugettext_lazy("Import ODK forms and instances.") + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): path = args[0] diff --git a/onadata/apps/viewer/management/commands/import_forms.py b/onadata/apps/viewer/management/commands/import_forms.py index aed6bc40ce..9be917d775 100644 --- a/onadata/apps/viewer/management/commands/import_forms.py +++ b/onadata/apps/viewer/management/commands/import_forms.py @@ -6,13 +6,13 @@ import os from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm class Command(BaseCommand): - help = ugettext_lazy("Import a folder of XForms for ODK.") + help = gettext_lazy("Import a folder of XForms for ODK.") def handle(self, *args, **kwargs): path = args[0] diff --git a/onadata/apps/viewer/management/commands/mark_start_times.py b/onadata/apps/viewer/management/commands/mark_start_times.py index e46624f02b..ecc2c7108e 100644 --- a/onadata/apps/viewer/management/commands/mark_start_times.py +++ b/onadata/apps/viewer/management/commands/mark_start_times.py @@ -1,11 +1,11 @@ from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.translation import gettext_lazy, gettext as _ from onadata.apps.viewer.models.data_dictionary import DataDictionary class Command(BaseCommand): - help = ugettext_lazy("This is a one-time command to " + help = gettext_lazy("This is a one-time command to " "mark start times of old surveys.") def handle(self, *args, **kwargs): diff --git a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py index 532bd05224..86c31b6fcd 100644 --- a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py +++ b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py @@ -1,12 +1,12 @@ from django.core.management.base import BaseCommand -from django.utils.translation import ugettext as _, ugettext_lazy +from django.utils.translation import gettext as _, gettext_lazy from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.utils.model_tools import queryset_iterator class Command(BaseCommand): - help = ugettext_lazy("Insert UUID into XML of all existing XForms") + help = gettext_lazy("Insert UUID into XML of all existing XForms") def handle(self, *args, **kwargs): self.stdout.write(_('%(nb)d XForms to update') diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 8b5942fed3..8a315507f5 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -10,7 +10,7 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from floip import FloipSurvey from kombu.exceptions import OperationalError from pyxform.builder import create_survey_element_from_dict diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 967bb75d77..4e28916c31 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -10,7 +10,7 @@ from django.db import models from django.db.models import JSONField from django.db.models.signals import post_delete -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.common_tags import OSM from onadata.libs.utils import async_status diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 66d8207ef7..1abe10543e 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -9,7 +9,7 @@ from django.conf import settings from django.db import connection, models from django.db.models.query import EmptyQuerySet -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import six from dateutil import parser diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index b5d260846d..438e10ca9b 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -26,7 +26,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from dpath import util as dpath_util from oauth2client import client as google_client diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index fb50c68014..e3478f1450 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -13,7 +13,7 @@ from django.core.signing import BadSignature from django.db import DataError from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import jwt from django_digest import HttpDigestAuthenticator @@ -250,18 +250,18 @@ def retrieve_user_identification(request) -> Tuple[Optional[str], Optional[str]] """ ip_address = None - if request.META.get("HTTP_X_REAL_IP"): - ip_address = request.META["HTTP_X_REAL_IP"].split(",")[0] + if request.headers.get('X-Real-Ip'): + ip_address = request.headers['X-Real-Ip'].split(",")[0] else: ip_address = request.META.get("REMOTE_ADDR") try: - if isinstance(request.META["HTTP_AUTHORIZATION"], bytes): + if isinstance(request.headers['Authorization'], bytes): username = ( - request.META["HTTP_AUTHORIZATION"].decode("utf-8").split('"')[1].strip() + request.headers['Authorization'].decode("utf-8").split('"')[1].strip() ) else: - username = request.META["HTTP_AUTHORIZATION"].split('"')[1].strip() + username = request.headers['Authorization'].split('"')[1].strip() except (TypeError, AttributeError, IndexError): pass else: diff --git a/onadata/libs/exceptions.py b/onadata/libs/exceptions.py index da0876fe25..c3b7103b4b 100644 --- a/onadata/libs/exceptions.py +++ b/onadata/libs/exceptions.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import APIException diff --git a/onadata/libs/models/textit_service.py b/onadata/libs/models/textit_service.py index 993062b870..f04fec2fd8 100644 --- a/onadata/libs/models/textit_service.py +++ b/onadata/libs/models/textit_service.py @@ -6,7 +6,7 @@ from onadata.apps.main.models.meta_data import MetaData from onadata.apps.restservice.models import RestService from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.db import IntegrityError from rest_framework import serializers diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 219abaa8f3..8a0604f617 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -13,7 +13,7 @@ from django.utils import timezone from django.utils.dateparse import parse_datetime -from django.utils.encoding import smart_text, force_str +from django.utils.encoding import smart_str, force_str from django.utils.xmlutils import SimplerXMLGenerator from six import iteritems from rest_framework import negotiation @@ -310,7 +310,7 @@ def _to_xml(self, xml, data): pass else: - xml.characters(smart_text(data)) + xml.characters(smart_str(data)) # pylint: disable=too-few-public-methods diff --git a/onadata/libs/serializers/clone_xform_serializer.py b/onadata/libs/serializers/clone_xform_serializer.py index 6822bf2bd5..763b1a5da8 100644 --- a/onadata/libs/serializers/clone_xform_serializer.py +++ b/onadata/libs/serializers/clone_xform_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.libs.models.clone_xform import CloneXForm diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index f09ad5eac8..006ef53e9f 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -6,7 +6,7 @@ from io import BytesIO from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers from rest_framework.reverse import reverse diff --git a/onadata/libs/serializers/dataview_serializer.py b/onadata/libs/serializers/dataview_serializer.py index c94359452d..0a0e323a39 100644 --- a/onadata/libs/serializers/dataview_serializer.py +++ b/onadata/libs/serializers/dataview_serializer.py @@ -1,6 +1,6 @@ import datetime -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.cache import cache from rest_framework import serializers diff --git a/onadata/libs/serializers/fields/organization_field.py b/onadata/libs/serializers/fields/organization_field.py index 7605db7af2..8c84a0314e 100644 --- a/onadata/libs/serializers/fields/organization_field.py +++ b/onadata/libs/serializers/fields/organization_field.py @@ -1,5 +1,5 @@ from builtins import str as text -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.apps.api.models.organization_profile import OrganizationProfile diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index 07febce5f3..190fc36931 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -1,5 +1,5 @@ from builtins import str as text -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.apps.logger.models.project import Project diff --git a/onadata/libs/serializers/floip_serializer.py b/onadata/libs/serializers/floip_serializer.py index a3477b6b57..07b05df0b0 100644 --- a/onadata/libs/serializers/floip_serializer.py +++ b/onadata/libs/serializers/floip_serializer.py @@ -15,7 +15,7 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Q from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ import six from floip import survey_to_floip_package diff --git a/onadata/libs/serializers/merged_xform_serializer.py b/onadata/libs/serializers/merged_xform_serializer.py index ae30de281f..c52c57509f 100644 --- a/onadata/libs/serializers/merged_xform_serializer.py +++ b/onadata/libs/serializers/merged_xform_serializer.py @@ -8,7 +8,7 @@ import uuid from django.db import transaction -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from pyxform.builder import create_survey_element_from_dict diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 605f5d36c9..983487c59a 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -12,7 +12,7 @@ from django.core.validators import URLValidator from django.db.utils import IntegrityError from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six.moves.urllib.parse import urlparse from rest_framework import serializers from rest_framework.reverse import reverse diff --git a/onadata/libs/serializers/note_serializer.py b/onadata/libs/serializers/note_serializer.py index 56a155ea86..8eb77c9003 100644 --- a/onadata/libs/serializers/note_serializer.py +++ b/onadata/libs/serializers/note_serializer.py @@ -2,7 +2,7 @@ """ Note Serializers Module """ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from guardian.shortcuts import assign_perm from rest_framework import exceptions, serializers diff --git a/onadata/libs/serializers/organization_member_serializer.py b/onadata/libs/serializers/organization_member_serializer.py index 4a0fb0baad..09df7131fe 100644 --- a/onadata/libs/serializers/organization_member_serializer.py +++ b/onadata/libs/serializers/organization_member_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.mail import send_mail from rest_framework import serializers diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index 6c0a78f837..2d5c8d7fab 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import User from django.db.models.query import QuerySet -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index c163c6b4fe..01ab28888c 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -11,7 +11,7 @@ from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_decode from django.utils.http import urlsafe_base64_encode -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from onadata.libs.utils.user_auth import invalidate_and_regen_tokens diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index d9f3f671cd..e0b79a88c3 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -8,7 +8,7 @@ from django.contrib.auth.models import User from django.core.cache import cache from django.db.utils import IntegrityError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers diff --git a/onadata/libs/serializers/share_project_serializer.py b/onadata/libs/serializers/share_project_serializer.py index 33118910fa..342d91eea3 100644 --- a/onadata/libs/serializers/share_project_serializer.py +++ b/onadata/libs/serializers/share_project_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.libs.models.share_project import ShareProject diff --git a/onadata/libs/serializers/share_team_project_serializer.py b/onadata/libs/serializers/share_team_project_serializer.py index bfedafc484..7a8d2351ab 100644 --- a/onadata/libs/serializers/share_team_project_serializer.py +++ b/onadata/libs/serializers/share_team_project_serializer.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.libs.models.share_team_project import ShareTeamProject diff --git a/onadata/libs/serializers/share_xform_serializer.py b/onadata/libs/serializers/share_xform_serializer.py index a9b65d72ee..0cc0592e9b 100644 --- a/onadata/libs/serializers/share_xform_serializer.py +++ b/onadata/libs/serializers/share_xform_serializer.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers from onadata.libs.models.share_xform import ShareXForm diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index ac329bd0eb..06d80d278e 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.core.cache import cache from django.conf import settings diff --git a/onadata/libs/serializers/tag_list_serializer.py b/onadata/libs/serializers/tag_list_serializer.py index 16ae5cdf13..205c013bb8 100644 --- a/onadata/libs/serializers/tag_list_serializer.py +++ b/onadata/libs/serializers/tag_list_serializer.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import serializers diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index 10570f9f16..8af5c82b03 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -11,7 +11,7 @@ from django.contrib.sites.models import Site from django.core.cache import cache from django.db import IntegrityError, transaction -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.utils import timezone import six diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index 516849f755..88d1ac63ae 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -5,7 +5,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.http import Http404 from django.urls import resolve, get_script_prefix, Resolver404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six.moves.urllib.parse import urlparse from guardian.shortcuts import get_users_with_perms diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index b381e72306..7f60420d61 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -14,7 +14,7 @@ from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.db.models import Count -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six.moves.urllib.parse import urlparse from six import itervalues from rest_framework import serializers diff --git a/onadata/libs/tests/utils/test_middleware.py b/onadata/libs/tests/utils/test_middleware.py index 1e3cb5d62f..b6c7ad8c01 100644 --- a/onadata/libs/tests/utils/test_middleware.py +++ b/onadata/libs/tests/utils/test_middleware.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock -from django.conf.urls import url +from django.urls import re_path from django.db import OperationalError from django.http import HttpResponse from django.test import RequestFactory, TestCase @@ -13,7 +13,7 @@ def normal_view(request): urlpatterns = [ - url('middleware_exceptions/view/', normal_view, name='normal'), + re_path('middleware_exceptions/view/', normal_view, name='normal'), ] diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index ea57ac72e3..b58e96cfdc 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -189,11 +189,11 @@ def track(user, event_name, properties=None, context=None, request=None): if request: context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id - context['receivedAt'] = request.META.get('HTTP_DATE', '') - context['userAgent'] = request.META.get('HTTP_USER_AGENT', '') + context['receivedAt'] = request.headers.get('Date', '') + context['userAgent'] = request.headers.get('User-Agent', '') context['campaign']['source'] = settings.HOSTNAME context['page']['path'] = request.path - context['page']['referrer'] = request.META.get('HTTP_REFERER', '') + context['page']['referrer'] = request.headers.get('Referer', '') context['page']['url'] = request.build_absolute_uri() if _segment: diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 65e942adcc..9398a1506b 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -14,7 +14,7 @@ from django.conf import settings from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from kombu.exceptions import OperationalError from oauth2client import client as google_client from oauth2client.client import ( diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index dab0426999..944b865dff 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # WE SHOULD PUT MORE STRUCTURE ON THESE TAGS SO WE CAN ACCESS DOCUMENT # FIELDS ELEGANTLY diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index 8c1e96b729..8c03242cf3 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -15,7 +15,7 @@ from django.conf import settings from django.core.mail import mail_admins from django.db import OperationalError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import six from raven.contrib.django.raven_compat.models import client diff --git a/onadata/libs/utils/country_field.py b/onadata/libs/utils/country_field.py index ee48e1ab67..6974a6cf23 100644 --- a/onadata/libs/utils/country_field.py +++ b/onadata/libs/utils/country_field.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # http://www.unece.org/cefact/locode/service/location.html diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index 9f30315913..6ea7293454 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -7,7 +7,7 @@ from django.conf import settings from django.db.models.query import QuerySet -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six import iteritems diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 1dac617129..484c11a925 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -24,7 +24,7 @@ from django.contrib.auth.models import User from django.core.files.storage import default_storage from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six import iteritems from multidb.pinning import use_master diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 37b7e405d3..eb11da3c75 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -21,7 +21,7 @@ from django.db.models.query import QuerySet from django.shortcuts import render from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from six.moves.urllib.parse import urlparse from six import iteritems from json2xlsclient.client import Client diff --git a/onadata/libs/utils/log.py b/onadata/libs/utils/log.py index 784edc1fba..8b2d470c19 100644 --- a/onadata/libs/utils/log.py +++ b/onadata/libs/utils/log.py @@ -1,7 +1,7 @@ import logging from datetime import datetime -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from onadata.libs.utils.viewer_tools import get_client_ip diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 0b236c1838..9207321335 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -26,7 +26,7 @@ from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.encoding import DjangoUnicodeDecodeError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from modilabs.utils.subprocess_timeout import ProcessTimedOut from multidb.pinning import use_master from pyxform.errors import PyXFormError diff --git a/onadata/libs/utils/middleware.py b/onadata/libs/utils/middleware.py index d7dfa139aa..25d6eaa7d5 100644 --- a/onadata/libs/utils/middleware.py +++ b/onadata/libs/utils/middleware.py @@ -7,7 +7,7 @@ from django.http import HttpResponseNotAllowed from django.template import loader from django.middleware.locale import LocaleMiddleware -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.utils.translation.trans_real import parse_accept_lang_header from multidb.pinning import use_master @@ -48,7 +48,7 @@ class LocaleMiddlewareWithTweaks(LocaleMiddleware): """ def process_request(self, request): - accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '') + accept = request.headers.get('Accept-Language', '') try: codes = [code for code, r in parse_accept_lang_header(accept)] if 'km' in codes and 'km-kh' not in codes: diff --git a/onadata/libs/utils/openid_connect_tools.py b/onadata/libs/utils/openid_connect_tools.py index cadf33fddf..73c8f48046 100644 --- a/onadata/libs/utils/openid_connect_tools.py +++ b/onadata/libs/utils/openid_connect_tools.py @@ -5,7 +5,7 @@ from django.http import HttpResponseRedirect, Http404 from django.core.cache import cache -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import jwt import requests diff --git a/onadata/libs/utils/quick_converter.py b/onadata/libs/utils/quick_converter.py index 45db46d9fa..8923c3be36 100644 --- a/onadata/libs/utils/quick_converter.py +++ b/onadata/libs/utils/quick_converter.py @@ -1,11 +1,11 @@ from django import forms -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from onadata.apps.viewer.models.data_dictionary import DataDictionary class QuickConverter(forms.Form): - xls_file = forms.FileField(label=ugettext_lazy("XLS File")) + xls_file = forms.FileField(label=gettext_lazy("XLS File")) def publish(self, user): if self.is_valid(): diff --git a/onadata/libs/utils/user_auth.py b/onadata/libs/utils/user_auth.py index 167645a955..77782b9a4d 100644 --- a/onadata/libs/utils/user_auth.py +++ b/onadata/libs/utils/user_auth.py @@ -147,7 +147,7 @@ def helper_auth_helper(request): return None # source, http://djangosnippets.org/snippets/243/ if 'HTTP_AUTHORIZATION' in request.META: - auth = request.META['HTTP_AUTHORIZATION'].split() + auth = request.headers['Authorization'].split() if len(auth) == 2 and auth[0].lower() == "basic": uname, passwd = base64.b64decode(auth[1].encode( 'utf-8')).decode('utf-8').split(':') diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index ce3e6e396c..ee3e2498cf 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -12,7 +12,7 @@ from django.conf import settings from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import requests from six.moves.urllib.parse import urljoin @@ -154,7 +154,7 @@ def get_client_ip(request): arguments: request -- HttpRequest object. """ - x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") + x_forwarded_for = request.headers.get('X-Forwarded-For') if x_forwarded_for: return x_forwarded_for.split(",")[0] @@ -321,7 +321,7 @@ def get_form_url( http_host = settings.TEST_HTTP_HOST username = settings.TEST_USERNAME else: - http_host = request.META.get("HTTP_HOST", "ona.io") + http_host = request.headers.get('Host', "ona.io") url = f"{protocol}://{http_host}" From 61da20e7a95e734fe40cf37b64f806664b24d2fa Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 17:23:24 +0300 Subject: [PATCH 129/234] Add prospector configuration file. --- .prospector.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .prospector.yaml diff --git a/.prospector.yaml b/.prospector.yaml new file mode 100644 index 0000000000..452c66da60 --- /dev/null +++ b/.prospector.yaml @@ -0,0 +1,6 @@ +strictness: medium +doc-warnings: false +test-warnings: false +autodetect: true +member-warnings: false +max-line-length: 88 From dbcf980d146f26492af4ac67efea0c24cc80a24e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 17:24:02 +0300 Subject: [PATCH 130/234] stats_serializer.py: cleanup --- onadata/libs/serializers/stats_serializer.py | 103 +++++++++++-------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index 06d80d278e..5684432fbd 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Stats API endpoint serializer. +""" from django.utils.translation import gettext as _ from django.core.cache import cache from django.conf import settings @@ -6,108 +10,125 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnList -from onadata.libs.data.statistics import\ - get_median_for_numeric_fields_in_form,\ - get_mean_for_numeric_fields_in_form,\ - get_mode_for_numeric_fields_in_form, get_min_max_range, get_all_stats +from onadata.libs.data.statistics import ( + get_median_for_numeric_fields_in_form, + get_mean_for_numeric_fields_in_form, + get_mode_for_numeric_fields_in_form, + get_min_max_range, + get_all_stats, +) from onadata.apps.logger.models.xform import XForm from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT -SELECT_FIELDS = ['select one', 'select multiple'] +SELECT_FIELDS = ["select one", "select multiple"] STATS_FUNCTIONS = { - 'mean': get_mean_for_numeric_fields_in_form, - 'median': get_median_for_numeric_fields_in_form, - 'mode': get_mode_for_numeric_fields_in_form, - 'range': get_min_max_range + "mean": get_mean_for_numeric_fields_in_form, + "median": get_median_for_numeric_fields_in_form, + "mode": get_mode_for_numeric_fields_in_form, + "range": get_min_max_range, } class SubmissionStatsSerializer(serializers.HyperlinkedModelSerializer): + """Submission stats serializer for use with the list API endpoint, summary of the + submission stats endpoints.""" + url = serializers.HyperlinkedIdentityField( - view_name='submissionstats-detail', lookup_field='pk') + view_name="submissionstats-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") +# pylint: disable=abstract-method class SubmissionStatsInstanceSerializer(serializers.Serializer): - def to_representation(self, obj): - if obj is None: - return super(SubmissionStatsInstanceSerializer, self)\ - .to_representation(obj) + """Submissions stats instance serializer - provides submission summary stats.""" + + def to_representation(self, instance): + """Returns submissions stats grouped by a specified field.""" + if instance is None: + return super().to_representation(instance) - request = self.context.get('request') - field = request.query_params.get('group') - name = request.query_params.get('name', field) + request = self.context.get("request") + field = request.query_params.get("group") + name = request.query_params.get("name", field) if field is None: - raise exceptions.ParseError(_(u"Expecting `group` and `name`" - u" query parameters.")) + raise exceptions.ParseError( + _("Expecting `group` and `name`" " query parameters.") + ) - cache_key = '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, - field, name) + cache_key = f"{XFORM_SUBMISSION_STAT}{instance.pk}{field}{name}" data = cache.get(cache_key) if data: return data try: - data = get_form_submissions_grouped_by_field( - obj, field, name) + data = get_form_submissions_grouped_by_field(instance, field, name) except ValueError as e: raise exceptions.ParseError(detail=e) else: if data: - element = obj.get_survey_element(field) + element = instance.get_survey_element(field) if element and element.type in SELECT_FIELDS: for record in data: - label = obj.get_choice_label(element, record[name]) + label = instance.get_choice_label(element, record[name]) record[name] = label - cache.set(cache_key, data, - settings.XFORM_SUBMISSION_STAT_CACHE_TIME) + cache.set(cache_key, data, settings.XFORM_SUBMISSION_STAT_CACHE_TIME) return data @property def data(self): - ret = super(serializers.Serializer, self).data + """Return the data as a list with ReturnList instead of a python object.""" + ret = super().data return ReturnList(ret, serializer=self) class StatsSerializer(serializers.HyperlinkedModelSerializer): + """Stats serializer for use with the list API endpoint, summary of the stats + endpoints.""" + url = serializers.HyperlinkedIdentityField( - view_name='stats-detail', lookup_field='pk') + view_name="stats-detail", lookup_field="pk" + ) class Meta: model = XForm - fields = ('id', 'id_string', 'url') + fields = ("id", "id_string", "url") +# pylint: disable=abstract-method class StatsInstanceSerializer(serializers.Serializer): - def to_representation(self, obj): - if obj is None: - return super(StatsInstanceSerializer, self).to_representation(obj) + """The stats instance serializer - calls the relevant statistical functions and + returns the results against form data submissions.""" + + def to_representation(self, instance): + """Returns the result of the selected stats function.""" + if instance is None: + return super().to_representation(instance) - request = self.context.get('request') - method = request.query_params.get('method', None) - field = request.query_params.get('field', None) + request = self.context.get("request") + method = request.query_params.get("method", None) + field = request.query_params.get("field", None) - if field and field not in obj.get_keys(): + if field and field not in instance.get_keys(): raise exceptions.ParseError(detail=_("Field not in XForm.")) - stats_function = STATS_FUNCTIONS.get(method and method.lower(), - get_all_stats) + stats_function = STATS_FUNCTIONS.get(method and method.lower(), get_all_stats) try: - data = stats_function(obj, field) + data = stats_function(instance, field) except ValueError as e: raise exceptions.ParseError(detail=e) From 8264a1a8748dd365246dfd343d1fc6bca8add396 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 19:27:34 +0300 Subject: [PATCH 131/234] Fix flake8 issues. --- .../commands/create_image_thumbnails.py | 70 +- .../management/commands/move_media_to_s3.py | 62 +- .../logger/management/commands/publish_xls.py | 55 +- .../management/commands/restore_backup.py | 18 +- .../commands/sync_deleted_instances_fix.py | 22 +- .../management/commands/update_moved_forms.py | 15 +- onadata/apps/main/registration_urls.py | 32 +- onadata/apps/main/urls.py | 742 +++++++++++------- .../management/commands/mark_start_times.py | 11 +- 9 files changed, 615 insertions(+), 412 deletions(-) diff --git a/onadata/apps/logger/management/commands/create_image_thumbnails.py b/onadata/apps/logger/management/commands/create_image_thumbnails.py index d0b83865a5..2b8a76e3be 100644 --- a/onadata/apps/logger/management/commands/create_image_thumbnails.py +++ b/onadata/apps/logger/management/commands/create_image_thumbnails.py @@ -16,52 +16,54 @@ class Command(BaseCommand): - help = gettext_lazy("Creates thumbnails for " - "all form images and stores them") + help = gettext_lazy("Creates thumbnails for " "all form images and stores them") def add_arguments(self, parser): parser.add_argument( - '-u', - '--username', - help=gettext_lazy("Username of the form user")) + "-u", "--username", help=gettext_lazy("Username of the form user") + ) parser.add_argument( - '-i', '--id_string', help=gettext_lazy("id string of the form")) + "-i", "--id_string", help=gettext_lazy("id string of the form") + ) parser.add_argument( - '-f', - '--force', - action='store_false', - help=gettext_lazy("regenerate thumbnails if they exist.")) + "-f", + "--force", + action="store_false", + help=gettext_lazy("regenerate thumbnails if they exist."), + ) def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( - 'instance', 'instance__xform') - if options.get('username'): - username = options.get('username') + "instance", "instance__xform" + ) + if options.get("username"): + username = options.get("username") try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( - "Error: username %(username)s does not exist" % - {'username': username}) + "Error: username %(username)s does not exist" + % {"username": username} + ) attachments_qs = attachments_qs.filter(instance__user=user) - if options.get('id_string'): - id_string = options.get('id_string') + if options.get("id_string"): + id_string = options.get("id_string") try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( - "Error: Form with id_string %(id_string)s does not exist" % - {'id_string': id_string}) + "Error: Form with id_string %(id_string)s does not exist" + % {"id_string": id_string} + ) attachments_qs = attachments_qs.filter(instance__xform=xform) - fs = get_storage_class('django.core.files.storage.FileSystemStorage')() + fs = get_storage_class("django.core.files.storage.FileSystemStorage")() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() - full_path = get_path(filename, - settings.THUMB_CONF['small']['suffix']) - if options.get('force') is not None: - for s in ['small', 'medium', 'large']: - fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) + full_path = get_path(filename, settings.THUMB_CONF["small"]["suffix"]) + if options.get("force") is not None: + for s in ["small", "medium", "large"]: + fp = get_path(filename, settings.THUMB_CONF[s]["suffix"]) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): @@ -70,17 +72,17 @@ def handle(self, *args, **options): resize(filename, att.extension) else: resize_local_env(filename, att.extension) - path = get_path( - filename, '%s' % THUMB_CONF['small']['suffix']) + path = get_path(filename, "%s" % THUMB_CONF["small"]["suffix"]) if default_storage.exists(path): self.stdout.write( - _(u'Thumbnails created for %(file)s') % - {'file': filename}) + _("Thumbnails created for %(file)s") % {"file": filename} + ) else: self.stdout.write( - _(u'Problem with the file %(file)s') % - {'file': filename}) + _("Problem with the file %(file)s") % {"file": filename} + ) except (IOError, OSError) as e: - self.stderr.write(_( - u'Error on %(filename)s: %(error)s') - % {'filename': filename, 'error': e}) + self.stderr.write( + _("Error on %(filename)s: %(error)s") + % {"filename": filename, "error": e} + ) diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index ff5d3772ab..08fed5b6da 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -6,54 +6,62 @@ from django.utils.translation import gettext as _, gettext_lazy from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.attachment import upload_to as\ - attachment_upload_to -from onadata.apps.logger.models.xform import XForm, upload_to as\ - xform_upload_to +from onadata.apps.logger.models.attachment import upload_to as attachment_upload_to +from onadata.apps.logger.models.xform import XForm, upload_to as xform_upload_to class Command(BaseCommand): - help = gettext_lazy("Moves all attachments and xls files " - "to s3 from the local file system storage.") + help = gettext_lazy( + "Moves all attachments and xls files " + "to s3 from the local file system storage." + ) def handle(self, *args, **kwargs): try: - fs = get_storage_class( - 'django.core.files.storage.FileSystemStorage')() - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() + fs = get_storage_class("django.core.files.storage.FileSystemStorage")() + s3 = get_storage_class("storages.backends.s3boto.S3BotoStorage")() except Exception: - self.stderr.write(_( - u"Missing necessary libraries. Try running: pip install -r" - "requirements/s3.pip")) + self.stderr.write( + _( + "Missing necessary libraries. Try running: pip install -r" + "requirements/s3.pip" + ) + ) sys.exit(1) default_storage = get_storage_class()() if default_storage.__class__ != s3.__class__: - self.stderr.write(_( - u"You must first set your default storage to s3 in your " - "local_settings.py file.")) + self.stderr.write( + _( + "You must first set your default storage to s3 in your " + "local_settings.py file." + ) + ) sys.exit(1) classes_to_move = [ - (Attachment, 'media_file', attachment_upload_to), - (XForm, 'xls', xform_upload_to), + (Attachment, "media_file", attachment_upload_to), + (XForm, "xls", xform_upload_to), ] for cls, file_field, upload_to in classes_to_move: - self.stdout.write(_( - u"Moving %(class)ss to s3...") % {'class': cls.__name__}) + self.stdout.write(_("Moving %(class)ss to s3...") % {"class": cls.__name__}) for i in cls.objects.all(): f = getattr(i, file_field) old_filename = f.name - if f.name and fs.exists(f.name) and not s3.exists( - upload_to(i, f.name)): + if f.name and fs.exists(f.name) and not s3.exists(upload_to(i, f.name)): f.save(fs.path(f.name), fs.open(fs.path(f.name))) - self.stdout.write(_( - "\t+ '%(fname)s'\n\t---> '%(url)s'") - % {'fname': fs.path(old_filename), 'url': f.url}) + self.stdout.write( + _("\t+ '%(fname)s'\n\t---> '%(url)s'") + % {"fname": fs.path(old_filename), "url": f.url} + ) else: self.stderr.write( "\t- (f.name=%s, fs.exists(f.name)=%s, not s3.exist" - "s(upload_to(i, f.name))=%s)" % ( - f.name, fs.exists(f.name), - not s3.exists(upload_to(i, f.name)))) + "s(upload_to(i, f.name))=%s)" + % ( + f.name, + fs.exists(f.name), + not s3.exists(upload_to(i, f.name)), + ) + ) diff --git a/onadata/apps/logger/management/commands/publish_xls.py b/onadata/apps/logger/management/commands/publish_xls.py index 34d323e359..d41479eaac 100644 --- a/onadata/apps/logger/management/commands/publish_xls.py +++ b/onadata/apps/logger/management/commands/publish_xls.py @@ -14,37 +14,40 @@ class Command(BaseCommand): - args = 'xls_file username project' - help = gettext_lazy("Publish an XLS file with the option of replacing an" - "existing one") + args = "xls_file username project" + help = gettext_lazy( + "Publish an XLS file with the option of replacing an" "existing one" + ) def add_arguments(self, parser): - parser.add_argument('xls_filepath') - parser.add_argument('username') + parser.add_argument("xls_filepath") + parser.add_argument("username") parser.add_argument( - '-p', '--project-name', action='store_true', dest='project_name') + "-p", "--project-name", action="store_true", dest="project_name" + ) parser.add_argument( - '-r', - '--replace', - action='store_true', - dest='replace', - help=gettext_lazy("Replace existing form if any")) + "-r", + "--replace", + action="store_true", + dest="replace", + help=gettext_lazy("Replace existing form if any"), + ) def handle(self, *args, **options): try: - xls_filepath = options['xls_filepath'] + xls_filepath = options["xls_filepath"] except KeyError: raise CommandError(_("You must provide the path to the xls file.")) # make sure path exists if not os.path.exists(xls_filepath): - raise CommandError( - _("The xls file '%s' does not exist.") % xls_filepath) + raise CommandError(_("The xls file '%s' does not exist.") % xls_filepath) try: - username = options['username'] + username = options["username"] except KeyError: raise CommandError( - _("You must provide the username to publish the form to.")) + _("You must provide the username to publish the form to.") + ) # make sure user exists try: user = User.objects.get(username=username) @@ -55,30 +58,34 @@ def handle(self, *args, **options): survey = create_survey_from_xls(xls_filepath) # check if a form with this id_string exists for this user - form_already_exists = XForm.objects.filter( - user=user, id_string=survey.id_string).count() > 0 + form_already_exists = ( + XForm.objects.filter(user=user, id_string=survey.id_string).count() > 0 + ) # id_string of form to replace, if any id_string = None if form_already_exists: - if 'replace' in options and options['replace']: + if "replace" in options and options["replace"]: id_string = survey.id_string self.stdout.write(_("Form already exist, replacing ..\n")) else: raise CommandError( - _("The form with id_string '%s' already exists, use the -r" - " option to replace it.") % survey.id_string) + _( + "The form with id_string '%s' already exists, use the -r" + " option to replace it." + ) + % survey.id_string + ) else: self.stdout.write(_("Form does NOT exist, publishing ..\n")) try: - project_name = options['project_name'] + project_name = options["project_name"] project = Project.objects.get(name=project_name) except (KeyError, Project.DoesNotExist): project = get_user_default_project(user) # publish - xls_file = django_file(xls_filepath, 'xls_file', - 'application/vnd.ms-excel') + xls_file = django_file(xls_filepath, "xls_file", "application/vnd.ms-excel") publish_xls_form(xls_file, user, project, id_string) self.stdout.write(_("Done..\n")) diff --git a/onadata/apps/logger/management/commands/restore_backup.py b/onadata/apps/logger/management/commands/restore_backup.py index 5e9484f908..4d03c0b37e 100644 --- a/onadata/apps/logger/management/commands/restore_backup.py +++ b/onadata/apps/logger/management/commands/restore_backup.py @@ -9,16 +9,16 @@ class Command(BaseCommand): - args = 'username input_file' - help = gettext_lazy("Restore a zip backup of a form and all its" - " submissions") + args = "username input_file" + help = gettext_lazy("Restore a zip backup of a form and all its" " submissions") def handle(self, *args, **options): try: username = args[0] except IndexError: - raise CommandError(_("You must provide the username to publish the" - " form to.")) + raise CommandError( + _("You must provide the username to publish the" " form to.") + ) # make sure user exists try: User.objects.get(username=username) @@ -32,7 +32,7 @@ def handle(self, *args, **options): else: input_file = os.path.realpath(input_file) - num_instances, num_restored = restore_backup_from_zip( - input_file, username) - sys.stdout.write("Restored %d of %d submissions\n" % - (num_restored, num_instances)) + num_instances, num_restored = restore_backup_from_zip(input_file, username) + sys.stdout.write( + "Restored %d of %d submissions\n" % (num_restored, num_instances) + ) diff --git a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py index d515edc7ca..f80f9682be 100644 --- a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py +++ b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py @@ -12,30 +12,32 @@ class Command(BaseCommand): - help = gettext_lazy("Fixes deleted instances by syncing " - "deleted items from mongo.") + help = gettext_lazy( + "Fixes deleted instances by syncing " "deleted items from mongo." + ) def handle(self, *args, **kwargs): # Reset all sql deletes to None - Instance.objects.exclude( - deleted_at=None, xform__downloadable=True).update(deleted_at=None) + Instance.objects.exclude(deleted_at=None, xform__downloadable=True).update( + deleted_at=None + ) # Get all mongo deletes - query = '{"$and": [{"_deleted_at": {"$exists": true}}, ' \ - '{"_deleted_at": {"$ne": null}}]}' + query = ( + '{"$and": [{"_deleted_at": {"$exists": true}}, ' + '{"_deleted_at": {"$ne": null}}]}' + ) query = json.loads(query) xform_instances = settings.MONGO_DB.instances cursor = xform_instances.find(query) for record in cursor: # update sql instance with deleted_at datetime from mongo try: - i = Instance.objects.get( - uuid=record["_uuid"], xform__downloadable=True) + i = Instance.objects.get(uuid=record["_uuid"], xform__downloadable=True) except Instance.DoesNotExist: continue else: deleted_at = parse_datetime(record["_deleted_at"]) if not timezone.is_aware(deleted_at): - deleted_at = timezone.make_aware( - deleted_at, timezone.utc) + deleted_at = timezone.make_aware(deleted_at, timezone.utc) i.set_deleted(deleted_at) diff --git a/onadata/apps/logger/management/commands/update_moved_forms.py b/onadata/apps/logger/management/commands/update_moved_forms.py index 6b7749fbe4..666c76c251 100644 --- a/onadata/apps/logger/management/commands/update_moved_forms.py +++ b/onadata/apps/logger/management/commands/update_moved_forms.py @@ -6,25 +6,26 @@ class Command(BaseCommand): - help = gettext_lazy("Ensures all the forms are owned by the project" - " owner") + help = gettext_lazy("Ensures all the forms are owned by the project" " owner") def handle(self, *args, **kwargs): - self.stdout.write("Updating forms owner", ending='\n') + self.stdout.write("Updating forms owner", ending="\n") for project in queryset_iterator(Project.objects.all()): for xform in project.xform_set.all(): try: if xform.user != project.organization: self.stdout.write( - "Processing: {} - {}".format(xform.id_string, - xform.user.username) + "Processing: {} - {}".format( + xform.id_string, xform.user.username + ) ) xform.user = project.organization xform.save() except Exception: self.stdout.write( - "Error processing: {} - {}".format(xform.id_string, - xform.user.username) + "Error processing: {} - {}".format( + xform.id_string, xform.user.username + ) ) pass diff --git a/onadata/apps/main/registration_urls.py b/onadata/apps/main/registration_urls.py index 42af15af0a..ff1eee5ece 100644 --- a/onadata/apps/main/registration_urls.py +++ b/onadata/apps/main/registration_urls.py @@ -16,23 +16,29 @@ from onadata.apps.main.forms import RegistrationFormUserProfile urlpatterns = [ - path('activate/complete/', - TemplateView.as_view( - template_name='registration/activation_complete.html'), - name='registration_activation_complete'), + path( + "activate/complete/", + TemplateView.as_view(template_name="registration/activation_complete.html"), + name="registration_activation_complete", + ), # Activation keys get matched by \w+ instead of the more specific # [a-fA-F0-9]{40} because a bad activation key should still get to the view # that way it can return a sensible "invalid key" message instead of a # confusing 404. - re_path(r'^activate/(?P\w+)/$', + re_path( + r"^activate/(?P\w+)/$", ActivationView.as_view(), - name='registration_activate'), - path('register/', + name="registration_activate", + ), + path( + "register/", FHRegistrationView.as_view(form_class=RegistrationFormUserProfile), - name='registration_register'), - path('register/complete/', - TemplateView.as_view( - template_name='registration/registration_complete.html'), - name='registration_complete'), - re_path(r'', include('registration.auth_urls')), + name="registration_register", + ), + path( + "register/complete/", + TemplateView.as_view(template_name="registration/registration_complete.html"), + name="registration_complete", + ), + re_path(r"", include("registration.auth_urls")), ] diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index 6f95fe300c..988889c5ab 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -6,339 +6,515 @@ from django.urls import include, re_path from django.conf.urls import i18n from django.contrib.staticfiles import views as staticfiles_views -from django.urls import re_path from django.views.generic import RedirectView from onadata.apps import sms_support from onadata.apps.api.urls.v1_urls import router as api_v1_router from onadata.apps.api.urls.v2_urls import router as api_v2_router from onadata.apps.api.urls.v1_urls import XFormListViewSet -from onadata.apps.api.viewsets.xform_list_viewset import ( - PreviewXFormListViewSet -) +from onadata.apps.api.viewsets.xform_list_viewset import PreviewXFormListViewSet from onadata.apps.api.urls.v1_urls import XFormSubmissionViewSet from onadata.apps.api.urls.v1_urls import BriefcaseViewset from onadata.apps.logger import views as logger_views from onadata.apps.main import views as main_views -from onadata.apps.main.registration_urls import ( - urlpatterns as registration_patterns -) +from onadata.apps.main.registration_urls import urlpatterns as registration_patterns from onadata.apps.restservice import views as restservice_views from onadata.apps.sms_support import views as sms_support_views from onadata.apps.viewer import views as viewer_views from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.libs.utils.analytics import init_analytics + # enable the admin: from django.contrib import admin -TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test' +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" admin.autodiscover() urlpatterns = [ # change Language - re_path(r'^i18n/', include(i18n)), - re_path('^api/v1/', include(api_v1_router.urls)), - re_path('^api/v2/', include(api_v2_router.urls)), + re_path(r"^i18n/", include(i18n)), + re_path("^api/v1/", include(api_v1_router.urls)), + re_path("^api/v2/", include(api_v2_router.urls)), # open id connect urls re_path(r"^", include("oidc.urls")), - re_path(r'^api-docs/', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), - re_path(r'^api/$', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), - re_path(r'^api/v1$', RedirectView.as_view(url='/api/v1/', permanent=True)), - + re_path( + r"^api-docs/", RedirectView.as_view(url=settings.STATIC_DOC, permanent=True) + ), + re_path(r"^api/$", RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), + re_path(r"^api/v1$", RedirectView.as_view(url="/api/v1/", permanent=True)), # django default stuff - re_path(r'^accounts/', include(registration_patterns)), - re_path(r'^admin/', admin.site.urls), - re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), - + re_path(r"^accounts/", include(registration_patterns)), + re_path(r"^admin/", admin.site.urls), + re_path(r"^admin/doc/", include("django.contrib.admindocs.urls")), # oath2_provider - re_path(r'^o/authorize/$', main_views.OnaAuthorizationView.as_view(), - name="oauth2_provider_authorize"), - re_path(r'^o/', include('oauth2_provider.urls', - namespace='oauth2_provider')), - + re_path( + r"^o/authorize/$", + main_views.OnaAuthorizationView.as_view(), + name="oauth2_provider_authorize", + ), + re_path(r"^o/", include("oauth2_provider.urls", namespace="oauth2_provider")), # main website views - re_path(r'^$', main_views.home), - re_path(r'^tutorial/$', main_views.tutorial, name='tutorial'), - re_path(r'^about-us/$', main_views.about_us, name='about-us'), - re_path(r'^getting_started/$', main_views.getting_started, - name='getting_started'), - re_path(r'^faq/$', main_views.faq, name='faq'), - re_path(r'^syntax/$', main_views.syntax, name='syntax'), - re_path(r'^privacy/$', main_views.privacy, name='privacy'), - re_path(r'^tos/$', main_views.tos, name='tos'), - re_path(r'^resources/$', main_views.resources, name='resources'), - re_path(r'^forms/$', main_views.form_gallery, name='forms_list'), - re_path(r'^forms/(?P[^/]+)$', main_views.show, name='form-show'), - re_path(r'^people/$', main_views.members_list, name='members-list'), - re_path(r'^xls2xform/$', main_views.xls2xform), - re_path(r'^support/$', main_views.support, name='support'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/stats$', - viewer_views.charts, name='form-stats'), - re_path(r'^login_redirect/$', main_views.login_redirect), - re_path(r'^attachment/$', viewer_views.attachment_url, - name='attachment_url'), - re_path(r'^attachment/(?P[^/]+)$', viewer_views.attachment_url, - name='attachment_url'), - re_path(r'^jsi18n/$', - django.views.i18n.JavaScriptCatalog.as_view( - packages=['onadata.apps.main', 'onadata.apps.viewer']), - name='javascript-catalog'), - re_path(r'^typeahead_usernames', main_views.username_list, - name='username_list'), - re_path(r'^(?P[^/]+)/$', main_views.profile, - name='user_profile'), - re_path(r'^(?P[^/]+)/profile$', main_views.public_profile, - name='public_profile'), - re_path(r'^(?P[^/]+)/settings', main_views.profile_settings, - name='profile-settings'), - re_path(r'^(?P[^/]+)/cloneform$', main_views.clone_xlsform, - name='clone-xlsform'), - re_path(r'^(?P[^/]+)/activity$', main_views.activity, - name='activity'), - re_path(r'^(?P[^/]+)/activity/api$', main_views.activity_api, - name='activity-api'), - re_path(r'^activity/fields$', main_views.activity_fields, - name='activity-fields'), - re_path(r'^(?P[^/]+)/api-token$', main_views.api_token, - name='api-token'), - + re_path(r"^$", main_views.home), + re_path(r"^tutorial/$", main_views.tutorial, name="tutorial"), + re_path(r"^about-us/$", main_views.about_us, name="about-us"), + re_path(r"^getting_started/$", main_views.getting_started, name="getting_started"), + re_path(r"^faq/$", main_views.faq, name="faq"), + re_path(r"^syntax/$", main_views.syntax, name="syntax"), + re_path(r"^privacy/$", main_views.privacy, name="privacy"), + re_path(r"^tos/$", main_views.tos, name="tos"), + re_path(r"^resources/$", main_views.resources, name="resources"), + re_path(r"^forms/$", main_views.form_gallery, name="forms_list"), + re_path(r"^forms/(?P[^/]+)$", main_views.show, name="form-show"), + re_path(r"^people/$", main_views.members_list, name="members-list"), + re_path(r"^xls2xform/$", main_views.xls2xform), + re_path(r"^support/$", main_views.support, name="support"), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/stats$", + viewer_views.charts, + name="form-stats", + ), + re_path(r"^login_redirect/$", main_views.login_redirect), + re_path(r"^attachment/$", viewer_views.attachment_url, name="attachment_url"), + re_path( + r"^attachment/(?P[^/]+)$", + viewer_views.attachment_url, + name="attachment_url", + ), + re_path( + r"^jsi18n/$", + django.views.i18n.JavaScriptCatalog.as_view( + packages=["onadata.apps.main", "onadata.apps.viewer"] + ), + name="javascript-catalog", + ), + re_path(r"^typeahead_usernames", main_views.username_list, name="username_list"), + re_path(r"^(?P[^/]+)/$", main_views.profile, name="user_profile"), + re_path( + r"^(?P[^/]+)/profile$", + main_views.public_profile, + name="public_profile", + ), + re_path( + r"^(?P[^/]+)/settings", + main_views.profile_settings, + name="profile-settings", + ), + re_path( + r"^(?P[^/]+)/cloneform$", + main_views.clone_xlsform, + name="clone-xlsform", + ), + re_path(r"^(?P[^/]+)/activity$", main_views.activity, name="activity"), + re_path( + r"^(?P[^/]+)/activity/api$", + main_views.activity_api, + name="activity-api", + ), + re_path(r"^activity/fields$", main_views.activity_fields, name="activity-fields"), + re_path(r"^(?P[^/]+)/api-token$", main_views.api_token, name="api-token"), # form specific - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)$', - main_views.show, - name='form-show'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/qrcode$', - main_views.qrcode, name='get_qrcode'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/api$', - main_views.api, name='mongo_view_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/public_api$', - main_views.public_api, name='public_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete_data$', - main_views.delete_data, name='delete_data'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/edit$', - main_views.edit, name='xform-edit'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/perms$', - main_views.set_perm, name='set-xform-permissions'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/photos', - main_views.form_photos, name='form-photos'), - - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/doc/(?P\d+)', # noqa - main_views.download_metadata, name='download-metadata'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete-doc/(?P\d+)', main_views.delete_metadata, # noqa - name='delete-metadata'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)', main_views.download_media_data, # noqa - name='download-media-data'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/addservice$', - restservice_views.add_service, name='add_restservice'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delservice$', - restservice_views.delete_service, - name='delete_restservice'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/update$', - main_views.update_xform, name='update-xform'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/preview$', - main_views.enketo_preview, name='enketo-preview'), - + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)$", + main_views.show, + name="form-show", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/qrcode$", + main_views.qrcode, + name="get_qrcode", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/api$", + main_views.api, + name="mongo_view_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/public_api$", + main_views.public_api, + name="public_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delete_data$", + main_views.delete_data, + name="delete_data", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/edit$", + main_views.edit, + name="xform-edit", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/perms$", + main_views.set_perm, + name="set-xform-permissions", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/photos", + main_views.form_photos, + name="form-photos", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/doc/(?P\d+)", # noqa + main_views.download_metadata, + name="download-metadata", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delete-doc/(?P\d+)", + main_views.delete_metadata, # noqa + name="delete-metadata", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)", + main_views.download_media_data, # noqa + name="download-media-data", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/addservice$", + restservice_views.add_service, + name="add_restservice", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/delservice$", + restservice_views.delete_service, + name="delete_restservice", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/update$", + main_views.update_xform, + name="update-xform", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/preview$", + main_views.enketo_preview, + name="enketo-preview", + ), # briefcase api urls - re_path(r'^(?P\w+)/view/submissionList$', - BriefcaseViewset.as_view({'get': 'list', 'head': 'list'}), - name='view-submission-list'), - re_path(r'^(?P\w+)/view/downloadSubmission$', - BriefcaseViewset.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='view-download-submission'), - re_path(r'^(?P\w+)/formUpload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='form-upload'), - re_path(r'^(?P\w+)/upload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='upload'), - + re_path( + r"^(?P\w+)/view/submissionList$", + BriefcaseViewset.as_view({"get": "list", "head": "list"}), + name="view-submission-list", + ), + re_path( + r"^(?P\w+)/view/downloadSubmission$", + BriefcaseViewset.as_view({"get": "retrieve", "head": "retrieve"}), + name="view-download-submission", + ), + re_path( + r"^(?P\w+)/formUpload$", + BriefcaseViewset.as_view({"post": "create", "head": "create"}), + name="form-upload", + ), + re_path( + r"^(?P\w+)/upload$", + BriefcaseViewset.as_view({"post": "create", "head": "create"}), + name="upload", + ), # exporting stuff - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv$', - viewer_views.data_export, name='csv_export', - kwargs={'export_type': 'csv'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.xls', - viewer_views.data_export, name='xls_export', - kwargs={'export_type': 'xls'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv.zip', - viewer_views.data_export, name='csv_zip_export', - kwargs={'export_type': 'csv_zip'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.sav.zip', - viewer_views.data_export, name='sav_zip_export', - kwargs={'export_type': 'sav_zip'}), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.kml$', - viewer_views.kml_export, name='kml-export'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.zip', - viewer_views.zip_export), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/gdocs$', - viewer_views.google_xls_export), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/map_embed', - viewer_views.map_embed_view), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/map', - viewer_views.map_view, name='map-view'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/instance', - viewer_views.instance, name='submission-instance'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/enter-data', - logger_views.enter_data, name='enter_data'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/add-submission-with', # noqa - viewer_views.add_submission_with, - name='add_submission_with'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/thank_you_submission', # noqa - viewer_views.thank_you_submission, - name='thank_you_submission'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/edit-data/(?P\d+)$', # noqa - logger_views.edit_data, name='edit_data'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/view-data', - viewer_views.data_view, name='data-view'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)/new$', # noqa - viewer_views.create_export, name='new-export'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/delete$', viewer_views.delete_export, name='delete-export'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/progress$', viewer_views.export_progress, - name='export-progress'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/$', viewer_views.export_list, name='export-list'), - re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/(?P[^/]+)$', - viewer_views.export_download, name='export-download'), - + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.csv$", + viewer_views.data_export, + name="csv_export", + kwargs={"export_type": "csv"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.xls", + viewer_views.data_export, + name="xls_export", + kwargs={"export_type": "xls"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.csv.zip", + viewer_views.data_export, + name="csv_zip_export", + kwargs={"export_type": "csv_zip"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.sav.zip", + viewer_views.data_export, + name="sav_zip_export", + kwargs={"export_type": "sav_zip"}, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.kml$", + viewer_views.kml_export, + name="kml-export", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/data\.zip", + viewer_views.zip_export, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/gdocs$", + viewer_views.google_xls_export, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/map_embed", + viewer_views.map_embed_view, + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/map", + viewer_views.map_view, + name="map-view", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/instance", + viewer_views.instance, + name="submission-instance", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/enter-data", + logger_views.enter_data, + name="enter_data", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/add-submission-with", # noqa + viewer_views.add_submission_with, + name="add_submission_with", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/thank_you_submission", # noqa + viewer_views.thank_you_submission, + name="thank_you_submission", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/edit-data/(?P\d+)$", # noqa + logger_views.edit_data, + name="edit_data", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/view-data", + viewer_views.data_view, + name="data-view", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)/new$", # noqa + viewer_views.create_export, + name="new-export", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/delete$", + viewer_views.delete_export, + name="delete-export", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/progress$", + viewer_views.export_progress, + name="export-progress", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/$", + viewer_views.export_list, + name="export-list", + ), + re_path( + r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" # noqa + "/(?P[^/]+)$", + viewer_views.export_download, + name="export-download", + ), # xform versions urls - re_path(r'^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)$', # noqa - XFormViewSet.as_view({'get': 'versions'}), - name='form-version-detail'), - re_path(r'^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)\.(?P[a-z0-9]+)/?$', # noqa - XFormViewSet.as_view({'get': 'versions'}), - name='form-version-detail'), - + re_path( + r"^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)$", # noqa + XFormViewSet.as_view({"get": "versions"}), + name="form-version-detail", + ), + re_path( + r"^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)\.(?P[a-z0-9]+)/?$", # noqa + XFormViewSet.as_view({"get": "versions"}), + name="form-version-detail", + ), # odk data urls - re_path(r'^submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^enketo/(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^enketo-preview/(?P\w+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/(?P\d+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^preview/(?P\w+)/(?P\d+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^preview/(?P\w+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), - re_path(r'^(?P\w+)/xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), - re_path(r'^xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), - re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', # noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$',# noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.' - '(?P([a-z]|[0-9])*)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), - re_path(r'^(?P\w+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^enketo/(?P\w+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^(?P\w+)/(?P\d+)/submission$', - XFormSubmissionViewSet.as_view( - {'post': 'create', 'head': 'create'}), - name='submissions'), - re_path(r'^(?P\w+)/bulk-submission$', - logger_views.bulksubmission), - re_path(r'^(?P\w+)/bulk-submission-form$', - logger_views.bulksubmission_form), - re_path(r'^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$', - XFormListViewSet.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='download_xform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xml$', - logger_views.download_xform, name='download_xform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xls$', - logger_views.download_xlsform, - name='download_xlsform'), - re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.json', - logger_views.download_jsonform, - name='download_jsonform'), - re_path(r'^(?P\w+)/delete/(?P[^/]+)/$', - logger_views.delete_xform, name='delete-xform'), - re_path(r'^(?P\w+)/(?P[^/]+)/toggle_downloadable/$', - logger_views.toggle_downloadable, name='toggle-downloadable'), - + re_path( + r"^submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^enketo/(?P\w+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^enketo-preview/(?P\w+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/(?P\d+)/formList$", + XFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^preview/(?P\w+)/(?P\d+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^preview/(?P\w+)/formList$", + PreviewXFormListViewSet.as_view({"get": "list", "head": "list"}), + name="form-list", + ), + re_path( + r"^(?P\w+)/xformsManifest/(?P[\d+^/]+)$", + XFormListViewSet.as_view({"get": "manifest", "head": "manifest"}), + name="manifest-url", + ), + re_path( + r"^xformsManifest/(?P[\d+^/]+)$", + XFormListViewSet.as_view({"get": "manifest", "head": "manifest"}), + name="manifest-url", + ), + re_path( + r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", # noqa + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$", # noqa + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\." + "(?P([a-z]|[0-9])*)$", + XFormListViewSet.as_view({"get": "media", "head": "media"}), + name="xform-media", + ), + re_path( + r"^(?P\w+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^enketo/(?P\w+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path( + r"^(?P\w+)/(?P\d+)/submission$", + XFormSubmissionViewSet.as_view({"post": "create", "head": "create"}), + name="submissions", + ), + re_path(r"^(?P\w+)/bulk-submission$", logger_views.bulksubmission), + re_path( + r"^(?P\w+)/bulk-submission-form$", logger_views.bulksubmission_form + ), + re_path( + r"^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$", + XFormListViewSet.as_view({"get": "retrieve", "head": "retrieve"}), + name="download_xform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.xml$", + logger_views.download_xform, + name="download_xform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.xls$", + logger_views.download_xlsform, + name="download_xlsform", + ), + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/form\.json", + logger_views.download_jsonform, + name="download_jsonform", + ), + re_path( + r"^(?P\w+)/delete/(?P[^/]+)/$", + logger_views.delete_xform, + name="delete-xform", + ), + re_path( + r"^(?P\w+)/(?P[^/]+)/toggle_downloadable/$", + logger_views.toggle_downloadable, + name="toggle-downloadable", + ), # SMS support - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', # noqa - sms_support.providers.import_submission_for_form, - name='sms_submission_form_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission$', - sms_support_views.import_submission_for_form, - name='sms_submission_form'), - re_path(r'^(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', - sms_support.providers.import_submission, - name='sms_submission_api'), - re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_multiple_submissions$', # noqa - sms_support_views.import_multiple_submissions_for_form, - name='sms_submissions_form'), - re_path(r'^(?P[^/]+)/sms_multiple_submissions$', - sms_support_views.import_multiple_submissions, - name='sms_submissions'), - re_path(r'^(?P[^/]+)/sms_submission$', - sms_support_views.import_submission, name='sms_submission'), - + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$", # noqa + sms_support.providers.import_submission_for_form, + name="sms_submission_form_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_submission$", + sms_support_views.import_submission_for_form, + name="sms_submission_form", + ), + re_path( + r"^(?P[^/]+)/sms_submission/(?P[a-z]+)/?$", + sms_support.providers.import_submission, + name="sms_submission_api", + ), + re_path( + r"^(?P[^/]+)/forms/(?P[^/]+)/sms_multiple_submissions$", # noqa + sms_support_views.import_multiple_submissions_for_form, + name="sms_submissions_form", + ), + re_path( + r"^(?P[^/]+)/sms_multiple_submissions$", + sms_support_views.import_multiple_submissions, + name="sms_submissions", + ), + re_path( + r"^(?P[^/]+)/sms_submission$", + sms_support_views.import_submission, + name="sms_submission", + ), # Stats tables - re_path(r'^(?P\w+)/forms/(?P[^/]+)/tables', - viewer_views.stats_tables, name='stats-tables'), - + re_path( + r"^(?P\w+)/forms/(?P[^/]+)/tables", + viewer_views.stats_tables, + name="stats-tables", + ), # static media - re_path(r'^media/(?P.*)$', django.views.static.serve, - {'document_root': settings.MEDIA_ROOT}), - re_path(r'^favicon\.ico', - RedirectView.as_view(url='/static/images/favicon.ico', - permanent=True)), - re_path(r'^static/(?P.*)$', staticfiles_views.serve), - + re_path( + r"^media/(?P.*)$", + django.views.static.serve, + {"document_root": settings.MEDIA_ROOT}, + ), + re_path( + r"^favicon\.ico", + RedirectView.as_view(url="/static/images/favicon.ico", permanent=True), + ), + re_path(r"^static/(?P.*)$", staticfiles_views.serve), # Health status - re_path(r'^status$', main_views.service_health) + re_path(r"^status$", main_views.service_health), ] -CUSTOM_URLS = getattr(settings, 'CUSTOM_MAIN_URLS', None) +CUSTOM_URLS = getattr(settings, "CUSTOM_MAIN_URLS", None) if CUSTOM_URLS: for url_module in CUSTOM_URLS: - urlpatterns.append(re_path(r'^', include(url_module))) + urlpatterns.append(re_path(r"^", include(url_module))) -if (settings.DEBUG or TESTING) and 'debug_toolbar' in settings.INSTALLED_APPS: +if (settings.DEBUG or TESTING) and "debug_toolbar" in settings.INSTALLED_APPS: try: import debug_toolbar except ImportError: pass else: urlpatterns += [ - re_path(r'^__debug__/', include(debug_toolbar.urls)), + re_path(r"^__debug__/", include(debug_toolbar.urls)), ] init_analytics() diff --git a/onadata/apps/viewer/management/commands/mark_start_times.py b/onadata/apps/viewer/management/commands/mark_start_times.py index ecc2c7108e..b6c4cd7286 100644 --- a/onadata/apps/viewer/management/commands/mark_start_times.py +++ b/onadata/apps/viewer/management/commands/mark_start_times.py @@ -5,8 +5,9 @@ class Command(BaseCommand): - help = gettext_lazy("This is a one-time command to " - "mark start times of old surveys.") + help = gettext_lazy( + "This is a one-time command to " "mark start times of old surveys." + ) def handle(self, *args, **kwargs): for dd in DataDictionary.objects.all(): @@ -14,6 +15,6 @@ def handle(self, *args, **kwargs): dd._mark_start_time_boolean() dd.save() except Exception: - self.stderr.write(_( - "Could not mark start time for DD: %(data)s") % { - 'data': repr(dd)}) + self.stderr.write( + _("Could not mark start time for DD: %(data)s") % {"data": repr(dd)} + ) From e3c2a03bc3b96f44b5322277d934389e7ac74814 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 19:53:14 +0300 Subject: [PATCH 132/234] Update test to set HTTP_ prefix header correctly for Django 3 --- onadata/libs/authentication.py | 10 +++++----- onadata/libs/tests/test_authentication.py | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index e3478f1450..739a6cffae 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -250,18 +250,18 @@ def retrieve_user_identification(request) -> Tuple[Optional[str], Optional[str]] """ ip_address = None - if request.headers.get('X-Real-Ip'): - ip_address = request.headers['X-Real-Ip'].split(",")[0] + if request.headers.get("X-Real-Ip"): + ip_address = request.headers["X-Real-Ip"].split(",")[0] else: ip_address = request.META.get("REMOTE_ADDR") try: - if isinstance(request.headers['Authorization'], bytes): + if isinstance(request.headers["Authorization"], bytes): username = ( - request.headers['Authorization'].decode("utf-8").split('"')[1].strip() + request.headers["Authorization"].decode("utf-8").split('"')[1].strip() ) else: - username = request.headers['Authorization'].split('"')[1].strip() + username = request.headers["Authorization"].split('"')[1].strip() except (TypeError, AttributeError, IndexError): pass else: diff --git a/onadata/libs/tests/test_authentication.py b/onadata/libs/tests/test_authentication.py index c7c1e4e956..caba609e8d 100644 --- a/onadata/libs/tests/test_authentication.py +++ b/onadata/libs/tests/test_authentication.py @@ -154,7 +154,10 @@ def test_check_lockout(self): # Uses X_REAL_IP if present self.assertNotIn("HTTP_X_REAL_IP", request.META) - request.META.update({"HTTP_X_REAL_IP": "1.2.3.4"}) + extra = {"HTTP_X_REAL_IP": "1.2.3.4"} + extra.update(self.extra) + request = self.factory.get("/", **extra) + self.assertEqual(check_lockout(request), ("1.2.3.4", "bob")) def test_exception_on_username_with_whitespaces(self): From 1832370da58fe11d51de4abb1b65bf1b8591e37b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 20:42:53 +0300 Subject: [PATCH 133/234] stats_serializer.py: fix super(serializers.Serializer, self) call --- .../api/tests/viewsets/test_stats_viewset.py | 232 +++++++++--------- onadata/libs/serializers/stats_serializer.py | 2 +- 2 files changed, 118 insertions(+), 116 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index 82cbe92484..83e395102d 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -8,256 +8,258 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.viewsets.stats_viewset import StatsViewSet -from onadata.apps.api.viewsets.submissionstats_viewset import\ - SubmissionStatsViewSet +from onadata.apps.api.viewsets.submissionstats_viewset import SubmissionStatsViewSet from onadata.apps.logger.models import XForm from onadata.libs.utils.logger_tools import publish_xml_form, create_instance from onadata.libs.utils.user_auth import get_user_default_project class TestStatsViewSet(TestBase): - def setUp(self): TestBase.setUp(self) self._create_user_and_login() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} - @patch('onadata.apps.logger.models.instance.submission_time') + @patch("onadata.apps.logger.models.instance.submission_time") def test_submissions_stats(self, mock_time): self._set_mock_time(mock_time) self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - 'id': formid, - 'id_string': u'transportation_2011_07_25', - 'url': 'http://testserver/api/v1/stats/submissions/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "url": "http://testserver/api/v1/stats/submissions/%s" % formid, + } + ] self.assertEqual(response.data, data) - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 400) - data = {u'detail': u'Expecting `group` and `name` query parameters.'} + data = {"detail": "Expecting `group` and `name` query parameters."} self.assertEqual(response.data, data) - request = self.factory.get('/?group=_xform_id_string', **self.extra) + request = self.factory.get("/?group=_xform_id_string", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = { - u'count': 4 - } - self.assertDictContainsSubset(data, response.data[0]) + data = {"count": 4} + self.assertDictContainsSubset(data, response.data[0], response.data) - request = self.factory.get('/?group=_submitted_by', **self.extra) + request = self.factory.get("/?group=_submitted_by", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = { - u'count': 4 - } + data = {"count": 4} self.assertDictContainsSubset(data, response.data[0]) - @patch('onadata.apps.logger.models.instance.submission_time') - def test_submissions_stats_with_xform_in_delete_async_queue( - self, mock_time): + @patch("onadata.apps.logger.models.instance.submission_time") + def test_submissions_stats_with_xform_in_delete_async_queue(self, mock_time): self._set_mock_time(mock_time) self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - 'id': formid, - 'id_string': u'transportation_2011_07_25', - 'url': 'http://testserver/api/v1/stats/submissions/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "transportation_2011_07_25", + "url": "http://testserver/api/v1/stats/submissions/%s" % formid, + } + ] self.assertEqual(response.data, data) initial_count = len(response.data) self.xform.deleted_at = timezone.now() self.xform.save() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), initial_count - 1) def test_form_list_select_one_choices_multi_language(self): - paths = [os.path.join( - self.this_directory, 'fixtures', 'good_eats_multilang', x) - for x in ['good_eats_multilang.xlsx', '1.xml']] + paths = [ + os.path.join(self.this_directory, "fixtures", "good_eats_multilang", x) + for x in ["good_eats_multilang.xlsx", "1.xml"] + ] self._publish_xls_file_and_set_xform(paths[0]) self._make_submission(paths[1]) - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/?group=rating', - **self.extra) + request = self.factory.get("/?group=rating", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = [{'count': 1, 'rating': u'Nothing Special'}] + data = [{"count": 1, "rating": "Nothing Special"}] self.assertEqual(data, response.data) def test_form_list_select_one_choices(self): self._tutorial_form_submission() - view = SubmissionStatsViewSet.as_view({'get': 'retrieve'}) + view = SubmissionStatsViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/?group=gender', **self.extra) + request = self.factory.get("/?group=gender", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) self.assertIsInstance(response.data, list) - data = [ - {'count': 2, 'gender': u'Female'}, - {'count': 1, 'gender': u'Male'} - ] - self.assertEqual(sorted(data, key=lambda k: k['gender']), - sorted(response.data, key=lambda k: k['gender'])) + data = [{"count": 2, "gender": "Female"}, {"count": 1, "gender": "Male"}] + self.assertEqual( + sorted(data, key=lambda k: k["gender"]), + sorted(response.data, key=lambda k: k["gender"]), + ) def test_anon_form_list(self): self._publish_transportation_form() self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + view = SubmissionStatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/") response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) def _tutorial_form_submission(self): tutorial_folder = os.path.join( - os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'tutorial') - self._publish_xls_file_and_set_xform(os.path.join(tutorial_folder, - 'tutorial.xlsx')) - instance_paths = [os.path.join(tutorial_folder, 'instances', i) - for i in ['1.xml', '2.xml', '3.xml']] + os.path.dirname(__file__), "..", "fixtures", "forms", "tutorial" + ) + self._publish_xls_file_and_set_xform( + os.path.join(tutorial_folder, "tutorial.xlsx") + ) + instance_paths = [ + os.path.join(tutorial_folder, "instances", i) + for i in ["1.xml", "2.xml", "3.xml"] + ] for path in instance_paths: - create_instance(self.user.username, open(path, 'rb'), []) + create_instance(self.user.username, open(path, "rb"), []) self.assertEqual(self.xform.instances.count(), 3) def _contributions_form_submissions(self): count = XForm.objects.count() - path = os.path.join(os.path.dirname(__file__), - '..', 'fixtures', 'forms', 'contributions') - form_path = os.path.join(path, 'contributions.xml') - with open(form_path, encoding='utf-8') as f: + path = os.path.join( + os.path.dirname(__file__), "..", "fixtures", "forms", "contributions" + ) + form_path = os.path.join(path, "contributions.xml") + with open(form_path, encoding="utf-8") as f: xml_file = ContentFile(f.read()) - xml_file.name = 'contributions.xml' + xml_file.name = "contributions.xml" project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) - instances_path = os.path.join(path, 'instances') + instances_path = os.path.join(path, "instances") for uuid in os.listdir(instances_path): - s_path = os.path.join(instances_path, uuid, 'submission.xml') - create_instance(self.user.username, open(s_path, 'rb'), []) + s_path = os.path.join(instances_path, uuid, "submission.xml") + create_instance(self.user.username, open(s_path, "rb"), []) self.assertEqual(self.xform.instances.count(), 6) def test_median_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=median', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=median", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 28.5, u'amount': 1100.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 28.5, "amount": 1100.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_mean_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=mean', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=mean", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 28.17, u'amount': 1455.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 28.17, "amount": 1455.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_mode_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=mode', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=mode", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': 24, u'amount': 430.0} - self.assertNotEqual(response.get('Cache-Control'), None) + data = {"age": 24, "amount": 430.0} + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_range_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=range', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=range", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - data = {u'age': {u'range': 10, u'max': 34, u'min': 24}, - u'amount': {u'range': 2770, u'max': 3200, u'min': 430}} - self.assertNotEqual(response.get('Cache-Control'), None) + data = { + "age": {"range": 10, "max": 34, "min": 24}, + "amount": {"range": 2770, "max": 3200, "min": 430}, + } + self.assertNotEqual(response.get("Cache-Control"), None) self.assertDictContainsSubset(data, response.data) def test_bad_field(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=median&field=INVALID', - **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=median&field=INVALID", **self.extra) formid = self.xform.pk response = view(request, pk=formid) self.assertEqual(response.status_code, 400) def test_all_stats_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/', **self.extra) + view = StatsViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) formid = self.xform.pk - data = [{ - u'id': formid, - u'id_string': u'contributions', - u'url': u'http://testserver/api/v1/stats/%s' % formid - }] + data = [ + { + "id": formid, + "id_string": "contributions", + "url": "http://testserver/api/v1/stats/%s" % formid, + } + ] self.assertEqual(data, response.data) - view = StatsViewSet.as_view({'get': 'retrieve'}) + view = StatsViewSet.as_view({"get": "retrieve"}) response = view(request, pk=formid) data = {} - data['age'] = { - 'mean': 28.17, - 'median': 28.5, - 'mode': 24, - 'max': 34, - 'min': 24, - 'range': 10 + data["age"] = { + "mean": 28.17, + "median": 28.5, + "mode": 24, + "max": 34, + "min": 24, + "range": 10, } - request = self.factory.get('/?field=age', **self.extra) + request = self.factory.get("/?field=age", **self.extra) age_response = view(request, pk=formid) self.assertEqual(data, age_response.data) - data['amount'] = { - 'mean': 1455, - 'median': 1100.0, - 'mode': 430, - 'max': 3200, - 'min': 430, - 'range': 2770 + data["amount"] = { + "mean": 1455, + "median": 1100.0, + "mode": 430, + "max": 3200, + "min": 430, + "range": 2770, } self.assertDictContainsSubset(data, response.data) def test_wrong_stat_function_api(self): self._contributions_form_submissions() - view = StatsViewSet.as_view({'get': 'retrieve'}) - request = self.factory.get('/?method=modes', **self.extra) + view = StatsViewSet.as_view({"get": "retrieve"}) + request = self.factory.get("/?method=modes", **self.extra) formid = self.xform.pk response = view(request, pk=formid) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index 5684432fbd..2db6c85f6f 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -90,7 +90,7 @@ def to_representation(self, instance): @property def data(self): """Return the data as a list with ReturnList instead of a python object.""" - ret = super().data + ret = super(serializers.Serializer, self).data return ReturnList(ret, serializer=self) From b6ac78260a229bd33661131939a7846959c69421 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 21:00:22 +0300 Subject: [PATCH 134/234] import_briefcase.py: cleanup --- .../management/commands/import_briefcase.py | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/onadata/apps/logger/management/commands/import_briefcase.py b/onadata/apps/logger/management/commands/import_briefcase.py index e203504fae..09c238a727 100644 --- a/onadata/apps/logger/management/commands/import_briefcase.py +++ b/onadata/apps/logger/management/commands/import_briefcase.py @@ -1,6 +1,13 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +import_briefcase command -from django.contrib.auth.models import User +- imports XForm XML from a folder and publishes the XForm. +- import XForm submissions XML from a folder and inserts into the table instances. +""" + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ @@ -8,21 +15,23 @@ class Command(BaseCommand): + """Insert all existing parsed instances into MongoDB""" + help = _("Insert all existing parsed instances into MongoDB") def add_arguments(self, parser): - parser.add_argument( - '--url', help=_("server url to pull forms and submissions")) - parser.add_argument('-u', '--username', help=_("Username")), - parser.add_argument('-p', '--password', help=_("Password")) - parser.add_argument('--to', help=_("username in this server")) + parser.add_argument("--url", help=_("server url to pull forms and submissions")) + parser.add_argument("-u", "--username", help=_("Username")) + parser.add_argument("-p", "--password", help=_("Password")) + parser.add_argument("--to", help=_("username in this server")) def handle(self, *args, **options): - url = options.get('url') - username = options.get('username') - password = options.get('password') - to = options.get('to') - user = User.objects.get(username=to) - bc = BriefcaseClient( - username=username, password=password, user=user, url=url) - bc.push() + """Insert all existing parsed instances into MongoDB""" + url = options.get("url") + username = options.get("username") + password = options.get("password") + user = get_user_model().objects.get(username=options.get("to")) + client = BriefcaseClient( + username=username, password=password, user=user, url=url + ) + client.push() From 736a497ff41942b9f3ca48a079070cecc9ce4c3b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 22:46:17 +0300 Subject: [PATCH 135/234] xform_viewset.py: cleanup --- onadata/apps/api/viewsets/xform_viewset.py | 189 ++++++++++++--------- 1 file changed, 113 insertions(+), 76 deletions(-) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 0c17a7ad73..8f4839a59e 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -1,12 +1,14 @@ +# -*- coding: utf-8 -*- import json import os import random from datetime import datetime import six +from six.moves.urllib.parse import urlparse from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.core.files.storage import default_storage @@ -19,11 +21,12 @@ HttpResponseRedirect, StreamingHttpResponse, ) +from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.http import urlencode from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache -from django.shortcuts import get_object_or_404 + from django_filters.rest_framework import DjangoFilterBackend try: @@ -31,8 +34,6 @@ except ImportError: pass -from onadata.apps.messaging.constants import XFORM, FORM_UPDATED -from onadata.apps.messaging.serializers import send_message from pyxform.builder import create_survey_element_from_dict from pyxform.xls2json import parse_file_to_json from rest_framework import exceptions, status @@ -42,15 +43,18 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet -from onadata.apps.api import tools as utils from onadata.apps.api import tasks +from onadata.apps.api import tools as utils from onadata.apps.api.permissions import XFormPermissions from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.xform import XForm, XFormUserObjectPermission from onadata.apps.logger.models.xform_version import XFormVersion from onadata.apps.logger.xform_instance_parser import XLSFormError +from onadata.apps.messaging.constants import FORM_UPDATED, XFORM +from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.export import Export from onadata.libs import authentication, filters +from onadata.libs.exceptions import EnketoError from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( AnonymousUserPublicFormsMixin, ) @@ -73,25 +77,24 @@ process_async_export, response_for_format, ) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.csv_import import ( get_async_csv_submission_status, + submission_xls_to_csv, submit_csv, submit_csv_async, - submission_xls_to_csv, ) from onadata.libs.utils.export_tools import parse_request_export_options from onadata.libs.utils.logger_tools import publish_form from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.string import str2bool from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, generate_enketo_form_defaults, + get_enketo_urls, get_form_url, ) -from onadata.libs.exceptions import EnketoError -from onadata.settings.common import XLS_EXTENSIONS, CSV_EXTENSION -from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete +from onadata.settings.common import CSV_EXTENSION, XLS_EXTENSIONS ENKETO_AUTH_COOKIE = getattr(settings, "ENKETO_AUTH_COOKIE", "__enketo") ENKETO_META_UID_COOKIE = getattr( @@ -101,14 +104,18 @@ settings, "ENKETO_META_USERNAME_COOKIE", "__enketo_meta_username" ) +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() +User = get_user_model() def upload_to_survey_draft(filename, username): + """Return the ``filename`` in the ``username`` survey-drafts directory.""" return os.path.join(username, "survey-drafts", os.path.split(filename)[1]) def get_survey_dict(csv_name): + """Returns the a CSV XLSForm file into a python object.""" survey_file = default_storage.open(csv_name, "rb") survey_dict = parse_file_to_json(survey_file.name, default_name="data") @@ -119,7 +126,7 @@ def get_survey_dict(csv_name): def _get_user(username): users = User.objects.filter(username__iexact=username) - return users.count() and users[0] or None + return users[0] if users.count() else None def _get_owner(request): @@ -129,14 +136,15 @@ def _get_owner(request): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError("User with username %s does not exist." % owner) - else: - owner = owner_obj + raise ValidationError(f"User with username {owner} does not exist.") + owner = owner_obj return owner def value_for_type(form, field, value): + """Returns a boolean value for the ``field`` of type ``BooleanField`` otherwise + returns the same ``value`` back.""" if form._meta.get_field(field).get_internal_type() == "BooleanField": return str2bool(value) @@ -164,10 +172,12 @@ def _try_update_xlsform(request, xform, owner): def result_has_error(result): + """Returns True if the ``result`` is a dict and has a type.""" return isinstance(result, dict) and result.get("type") def get_survey_xml(csv_name): + """Creates and returns the XForm XML from a CSV XLSform.""" survey_dict = get_survey_dict(csv_name) survey = create_survey_element_from_dict(survey_dict) return survey.to_xml() @@ -176,7 +186,7 @@ def get_survey_xml(csv_name): def set_enketo_signed_cookies(resp, username=None, json_web_token=None): """Set signed cookies for JWT token in the HTTPResponse resp object.""" if not username and not json_web_token: - return + return None max_age = 30 * 24 * 60 * 60 * 1000 enketo_meta_uid = {"max_age": max_age, "salt": settings.ENKETO_API_SALT} @@ -203,7 +213,7 @@ def parse_webform_return_url(return_url, request): this data or data in the request. Construct a proper return URL, which has stripped the authentication data, to return the user. """ - from six.moves.urllib.parse import urlparse + jwt_param = None url = urlparse(return_url) try: @@ -212,17 +222,17 @@ def parse_webform_return_url(return_url, request): jwt_param = jwt_param and jwt_param[0].split("=")[1] if not jwt_param: - return + return None except IndexError: pass if "/_/" in return_url: # offline url - redirect_url = "%s://%s%s#%s" % (url.scheme, url.netloc, url.path, url.fragment) + redirect_url = f"{url.scheme}://{url.netloc}{url.path}#{url.fragment}" elif "/::" in return_url: # non-offline url - redirect_url = "%s://%s%s" % (url.scheme, url.netloc, url.path) + redirect_url = f"{url.scheme}://{url.netloc}{url.path}" else: # unexpected format - return + return None response_redirect = HttpResponseRedirect(redirect_url) @@ -243,8 +253,10 @@ def parse_webform_return_url(return_url, request): ) return response_redirect + return None +# pylint: disable=too-many-ancestors class XFormViewSet( AnonymousUserPublicFormsMixin, CacheControlMixin, @@ -341,7 +353,7 @@ def get_serializer_class(self): if self.action == "list": return XFormBaseSerializer - return super(XFormViewSet, self).get_serializer_class() + return super().get_serializer_class() def create(self, request, *args, **kwargs): try: @@ -371,7 +383,8 @@ def create_async(self, request, *args, **kwargs): resp_code = status.HTTP_400_BAD_REQUEST if request.method == "GET": - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" survey = tasks.get_async_status(request.query_params.get("job_uuid")) if "pk" in survey: @@ -416,21 +429,26 @@ def create_async(self, request, *args, **kwargs): @action(methods=["GET", "HEAD"], detail=True) @never_cache - def form(self, request, format="json", **kwargs): + def form(self, request, **kwargs): + """Returns the XLSForm in any of JSON, XML, XLS(X), CSV formats.""" form = self.get_object() - if format not in ["json", "xml", "xls", "xlsx", "csv"]: + form_format = kwargs.get("format", "json") + if form_format not in ["json", "xml", "xls", "xlsx", "csv"]: return HttpResponseBadRequest( "400 BAD REQUEST", content_type="application/json", status=400 ) - self.etag_data = "{}".format(form.date_modified) - filename = form.id_string + "." + format - response = response_for_format(form, format=format) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{form.date_modified}" + filename = form.id_string + "." + form_format + response = response_for_format(form, format=form_format) response["Content-Disposition"] = "attachment; filename=" + filename return response + # pylint: disable=no-self-use @action(methods=["GET"], detail=False) def login(self, request, **kwargs): + """Authenticate and redirect to URL in `return` query parameter.""" return_url = request.query_params.get("return") if return_url: @@ -453,6 +471,7 @@ def login(self, request, **kwargs): def enketo(self, request, **kwargs): """Expose enketo urls.""" survey_type = self.kwargs.get("survey_type") or request.GET.get("survey_type") + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() form_url = get_form_url( request, @@ -474,7 +493,7 @@ def enketo(self, request, **kwargs): preview_url = enketo_urls.get("preview_url") single_submit_url = enketo_urls.get("single_url") except EnketoError as e: - data = {"message": _("Enketo error: %s" % e)} + data = {"message": _(f"Enketo error: {e}")} else: if survey_type == "single": http_status = status.HTTP_200_OK @@ -489,8 +508,10 @@ def enketo(self, request, **kwargs): return Response(data, http_status) + # pylint: disable=unused-argument @action(methods=["POST", "GET"], detail=False) def survey_preview(self, request, **kwargs): + """Handle survey preview XLSForms.""" username = request.user.username if request.method.upper() == "POST": if not username: @@ -498,9 +519,10 @@ def survey_preview(self, request, **kwargs): csv_data = request.data.get("body") if csv_data: - rand_name = "survey_draft_%s.csv" % "".join( + random_name = "".join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6) ) + rand_name = f"survey_draft_{random_name}.csv" csv_file = ContentFile(csv_data) csv_name = default_storage.save( upload_to_survey_draft(rand_name, username), csv_file @@ -514,34 +536,33 @@ def survey_preview(self, request, **kwargs): return Response( {"unique_string": rand_name, "username": username}, status=200 ) - else: - raise ParseError("Missing body") + raise ParseError("Missing body") - if request.method.upper() == "GET": - filename = request.query_params.get("filename") - username = request.query_params.get("username") + filename = request.query_params.get("filename") + username = request.query_params.get("username") - if not username: - raise ParseError("Username not provided") - if not filename: - raise ParseError("Filename MUST be provided") + if not username: + raise ParseError("Username not provided") + if not filename: + raise ParseError("Filename MUST be provided") - csv_name = upload_to_survey_draft(filename, username) + csv_name = upload_to_survey_draft(filename, username) + result = publish_form(lambda: get_survey_xml(csv_name)) + if result_has_error(result): + raise ParseError(result.get("text")) - result = publish_form(lambda: get_survey_xml(csv_name)) + # pylint: disable=attribute-defined-outside-init + self.etag_data = result - if result_has_error(result): - raise ParseError(result.get("text")) - - self.etag_data = result - - return Response(result, status=200) + return Response(result, status=200) def retrieve(self, request, *args, **kwargs): + """Returns a forms properties.""" lookup_field = self.lookup_field lookup = self.kwargs.get(lookup_field) if lookup == self.public_forms_endpoint: + # pylint: disable=attribute-defined-outside-init self.object_list = self._get_public_forms_queryset() page = self.paginate_queryset(self.object_list) @@ -560,12 +581,14 @@ def retrieve(self, request, *args, **kwargs): if export_type is None or export_type in ["json", "debug"]: # perform default viewset retrieve, no data export - return super(XFormViewSet, self).retrieve(request, *args, **kwargs) + return super().retrieve(request, *args, **kwargs) return custom_response_handler(request, xform, query, export_type, token, meta) @action(methods=["POST"], detail=True) def share(self, request, *args, **kwargs): + """Perform form sharing.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() usernames_str = request.data.get("usernames", request.data.get("username")) @@ -591,6 +614,8 @@ def share(self, request, *args, **kwargs): @action(methods=["POST"], detail=True) def clone(self, request, *args, **kwargs): + """Clone/duplicate an existing form.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() data = {"xform": self.object.pk, "username": request.data.get("username")} project = request.data.get("project_id") @@ -600,19 +625,20 @@ def clone(self, request, *args, **kwargs): if serializer.is_valid(): clone_to_user = User.objects.get(username=data["username"]) if not request.user.has_perm("can_add_xform", clone_to_user.profile): + user = request.user.username + account = data["username"] raise exceptions.PermissionDenied( detail=_( - "User %(user)s has no permission to add " - "xforms to account %(account)s" - % {"user": request.user.username, "account": data["username"]} + f"User {user} has no permission to add " + f"xforms to account {account}" ) ) try: xform = serializer.save() - except IntegrityError: + except IntegrityError as e: raise ParseError( "A clone with the same id_string has already been created" - ) + ) from e serializer = XFormSerializer( xform.cloned_form, context={"request": request} ) @@ -634,6 +660,7 @@ def data_import(self, request, *args, **kwargs): for POST request passing the `request.FILES.get('xls_file')` upload for import if xls_file is provided instead of csv_file """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() resp = {} if request.method == "GET": @@ -644,14 +671,14 @@ def data_import(self, request, *args, **kwargs): ) ) self.last_modified_date = timezone.now() - except ValueError: + except ValueError as e: raise ParseError( ( "The instance of the result is not a " "basestring; the job_uuid variable might " "be incorrect" ) - ) + ) from e else: csv_file = request.FILES.get("csv_file", None) xls_file = request.FILES.get("xls_file", None) @@ -669,7 +696,7 @@ def data_import(self, request, *args, **kwargs): if xls_file and xls_file.name.split(".")[-1] in XLS_EXTENSIONS: csv_file = submission_xls_to_csv(xls_file) overwrite = request.query_params.get("overwrite") - overwrite = True if overwrite and overwrite.lower() == "true" else False + overwrite = overwrite.lower() == "true" size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD try: csv_size = csv_file.size @@ -692,8 +719,7 @@ def data_import(self, request, *args, **kwargs): ) if task is None: raise ParseError("Task not found") - else: - resp.update({"task_id": task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, @@ -712,6 +738,7 @@ def csv_import(self, request, *args, **kwargs): for GET requests passing `job_uuid` query param for job progress polling """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() resp = {} if request.method == "GET": @@ -721,15 +748,16 @@ def csv_import(self, request, *args, **kwargs): request.query_params.get("job_uuid") ) ) + # pylint: disable=attribute-defined-outside-init self.last_modified_date = timezone.now() - except ValueError: + except ValueError as e: raise ParseError( ( "The instance of the result is not a " "basestring; the job_uuid variable might " "be incorrect" ) - ) + ) from e else: csv_file = request.FILES.get("csv_file", None) if csv_file is None: @@ -738,7 +766,7 @@ def csv_import(self, request, *args, **kwargs): resp.update({"error": "csv_file not a csv file"}) else: overwrite = request.query_params.get("overwrite") - overwrite = True if overwrite and overwrite.lower() == "true" else False + overwrite = overwrite.lower() == "true" size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: resp.update( @@ -757,8 +785,7 @@ def csv_import(self, request, *args, **kwargs): ) if task is None: raise ParseError("Task not found") - else: - resp.update({"task_id": task.task_id}) + resp.update({"task_id": task.task_id}) return Response( data=resp, @@ -768,6 +795,8 @@ def csv_import(self, request, *args, **kwargs): ) def partial_update(self, request, *args, **kwargs): + """Partial update of a form's properties.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() owner = self.object.user @@ -778,12 +807,13 @@ def partial_update(self, request, *args, **kwargs): return _try_update_xlsform(request, self.object, owner) try: - return super(XFormViewSet, self).partial_update(request, *args, **kwargs) + return super().partial_update(request, *args, **kwargs) except XLSFormError as e: - raise ParseError(str(e)) + raise ParseError(str(e)) from e @action(methods=["DELETE", "GET"], detail=True) def delete_async(self, request, *args, **kwargs): + """Delete asynchronous endpoint `/api/v1/forms/{pk}/delete_async`.""" if request.method == "DELETE": xform = self.get_object() resp = { @@ -801,11 +831,13 @@ def delete_async(self, request, *args, **kwargs): job_uuid = request.query_params.get("job_uuid") resp = tasks.get_async_status(job_uuid) resp_code = status.HTTP_202_ACCEPTED - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" return Response(data=resp, status=resp_code) def destroy(self, request, *args, **kwargs): + """Soft deletes a form - `DELETE /api/v1/forms/{pk}`""" xform = self.get_object() user = request.user xform.soft_delete(user=user) @@ -814,23 +846,25 @@ def destroy(self, request, *args, **kwargs): @action(methods=["GET"], detail=True) def versions(self, request, *args, **kwargs): + """Returns all form versions.""" xform = self.get_object() version_id = kwargs.get("version_id") requested_format = kwargs.get("format") or "json" - if not version_id: - queryset = XFormVersion.objects.filter(xform=xform) - serializer = XFormVersionListSerializer( - queryset, many=True, context={"request": request} - ) - return Response(data=serializer.data, status=status.HTTP_200_OK) - if version_id: version = get_object_or_404(XFormVersion, version=version_id, xform=xform) return response_for_format(version, format=requested_format) + queryset = XFormVersion.objects.filter(xform=xform) + serializer = XFormVersionListSerializer( + queryset, many=True, context={"request": request} + ) + + return Response(data=serializer.data, status=status.HTTP_200_OK) + @action(methods=["GET"], detail=True) def export_async(self, request, *args, **kwargs): + """Returns the status of an async export.""" job_uuid = request.query_params.get("job_uuid") export_type = request.query_params.get("format") query = request.query_params.get("query") @@ -880,7 +914,8 @@ def export_async(self, request, *args, **kwargs): content_type="application/json", ) - self.etag_data = "{}".format(timezone.now()) + # pylint: disable=attribute-defined-outside-init + self.etag_data = f"{timezone.now()}" return Response( data=resp, status=status.HTTP_202_ACCEPTED, content_type="application/json" @@ -902,6 +937,7 @@ def get_json_string(item): ).data ) + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(queryset, get_json_string), content_type="application/json" ) @@ -919,19 +955,20 @@ def get_json_string(item): return response def list(self, request, *args, **kwargs): - STREAM_DATA = getattr(settings, "STREAM_DATA", False) + stream_data = getattr(settings, "STREAM_DATA", False) try: queryset = self.filter_queryset(self.get_queryset()) last_modified = queryset.values_list("date_modified", flat=True).order_by( "-date_modified" ) + # pylint: disable=attribute-defined-outside-init if last_modified: self.etag_data = last_modified[0].isoformat() - if STREAM_DATA: + if stream_data: self.object_list = queryset resp = self._get_streaming_response() else: - resp = super(XFormViewSet, self).list(request, *args, **kwargs) + resp = super().list(request, *args, **kwargs) except XLSFormError as e: resp = HttpResponseBadRequest(e) From 6ee49f330275205f15514dbaa923750258f6c377 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 22:58:42 +0300 Subject: [PATCH 136/234] Update test to set HTTP_ prefix header correctly for Django 3 --- onadata/apps/api/tests/viewsets/test_abstract_viewset.py | 5 +++-- onadata/apps/api/tests/viewsets/test_connect_viewset.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index c958c7a54d..b4a1e8b7a1 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -629,13 +629,14 @@ def is_sorted_asc(self, s): return self.is_sorted_asc(s[1:]) return False - def _get_request_session_with_auth(self, view, auth): + def _get_request_session_with_auth(self, view, auth, extra=None): request = self.factory.head("/") response = view(request) self.assertTrue(response.has_header("WWW-Authenticate")) self.assertTrue(response["WWW-Authenticate"].startswith("Digest ")) self.assertIn("nonce=", response["WWW-Authenticate"]) - request = self.factory.get("/") + extra = {} if extra is None else extra + request = self.factory.get("/", **extra) request.META.update(auth(request.META, response)) request.session = self.client.session diff --git a/onadata/apps/api/tests/viewsets/test_connect_viewset.py b/onadata/apps/api/tests/viewsets/test_connect_viewset.py index 0ede55e361..ae3053e51d 100644 --- a/onadata/apps/api/tests/viewsets/test_connect_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_connect_viewset.py @@ -456,8 +456,10 @@ def test_login_attempts(self, send_account_lockout_email): ) self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) + request = self._get_request_session_with_auth( + view, auth, extra={"HTTP_X_REAL_IP": "5.6.7.8"} + ) # login attempts are tracked separately for other IPs - request.META.update({"HTTP_X_REAL_IP": "5.6.7.8"}) response = view(request) self.assertEqual(response.status_code, 401) self.assertEqual(cache.get(safe_key(f"login_attempts-{request_ip}-bob")), 2) From dda19949087fdc1289ae740f7419fad471696329 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 23:04:25 +0300 Subject: [PATCH 137/234] import.py: cleanup --- onadata/apps/viewer/management/commands/import.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/onadata/apps/viewer/management/commands/import.py b/onadata/apps/viewer/management/commands/import.py index fe1894423c..64a04925f9 100644 --- a/onadata/apps/viewer/management/commands/import.py +++ b/onadata/apps/viewer/management/commands/import.py @@ -1,5 +1,7 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding: utf-8 -*- +"""import command - Combines and runs import_forms and import_instances commands""" import os from django.core.management.base import BaseCommand @@ -8,9 +10,12 @@ class Command(BaseCommand): + """Import ODK forms and instances.""" + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): + """Import ODK forms and instances.""" path = args[0] - call_command('import_forms', os.path.join(path, "forms")) - call_command('import_instances', os.path.join(path, "instances")) + call_command("import_forms", os.path.join(path, "forms")) + call_command("import_instances", os.path.join(path, "instances")) From 34396c3540e5f7271c128b65d68ee65c52ac455f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 23:36:43 +0300 Subject: [PATCH 138/234] logger/views.py: cleanup --- onadata/apps/logger/views.py | 95 +++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index dca577b202..41f9a3dc4b 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -2,10 +2,8 @@ """ logger views. """ -import json import os import tempfile -from builtins import str as text from datetime import datetime import pytz @@ -13,7 +11,7 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.sites.models import Site from django.core.files import File from django.core.files.storage import get_storage_class @@ -22,6 +20,7 @@ HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect, + JsonResponse, ) from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader @@ -62,18 +61,21 @@ IO_ERROR_STRINGS = ["request data read error", "error during read(65536) on wsgi.input"] +# pylint: disable=invalid-name +User = get_user_model() + def _bad_request(e): - strerror = text(e) + strerror = str(e) return strerror and strerror in IO_ERROR_STRINGS -def _extract_uuid(text): - text = text[text.find("@key=") : -1].replace("@key=", "") - if text.startswith("uuid:"): - text = text.replace("uuid:", "") - return text +def _extract_uuid(input_string): + input_string = input_string[input_string.find("@key=") : -1].replace("@key=", "") + if input_string.startswith("uuid:"): + input_string = input_string.replace("uuid:", "") + return input_string def _parse_int(num): @@ -81,6 +83,7 @@ def _parse_int(num): return num and int(num) except ValueError: pass + return None def _html_submission_response(request, instance): @@ -97,7 +100,7 @@ def _submission_response(instance): data["message"] = _("Successful submission.") data["formid"] = instance.xform.id_string data["encrypted"] = instance.xform.encrypted - data["instanceID"] = "uuid:%s" % instance.uuid + data["instanceID"] = f"uuid:{instance.uuid}" data["submissionDate"] = instance.date_created.isoformat() data["markedAsCompleteDate"] = instance.date_modified.isoformat() @@ -163,7 +166,7 @@ def bulksubmission(request, username): "success": success_count, "rejected": total_count - success_count, }, - "errors": "%d %s" % (len(errors), errors), + "errors": f"{len(errors)} {errors}", } audit = {"bulk_submission_log": json_msg} audit_log( @@ -174,7 +177,7 @@ def bulksubmission(request, username): audit, request, ) - response = HttpResponse(json.dumps(json_msg)) + response = JsonResponse(json_msg) response.status_code = 200 response["Location"] = request.build_absolute_uri(request.path) return response @@ -189,11 +192,12 @@ def bulksubmission_form(request, username=None): if request.user.username == username: return render(request, "bulk_submission_form.html") - return HttpResponseRedirect("/%s" % request.user.username) + return HttpResponseRedirect(f"/{request.user.username}") +# pylint: disable=invalid-name @require_GET -def formList(request, username): # pylint: disable=C0103 +def formList(request, username): """ formList view, /formList OpenRosa Form Discovery API 1.0. """ @@ -253,8 +257,9 @@ def formList(request, username): # pylint: disable=C0103 return response +# pylint: disable=invalid-name @require_GET -def xformsManifest(request, username, id_string): # pylint: disable=C0103 +def xformsManifest(request, username, id_string): """ XFormManifest view, part of OpenRosa Form Discovery API 1.0. """ @@ -290,9 +295,11 @@ def xformsManifest(request, username, id_string): # pylint: disable=C0103 return response +# pylint: disable=too-many-return-statements +# pylint: disable=too-many-branches @require_http_methods(["HEAD", "POST"]) @csrf_exempt -def submission(request, username=None): # pylint: disable=R0911,R0912 +def submission(request, username=None): """ Submission view, /submission of the OpenRosa Form Submission API 1.0. """ @@ -309,7 +316,7 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 response = OpenRosaResponse(status=204) if username: response["Location"] = request.build_absolute_uri().replace( - request.get_full_path(), "/%s/submission" % username + request.get_full_path(), f"/{username}/submission" ) else: response["Location"] = request.build_absolute_uri().replace( @@ -340,7 +347,7 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 if error: return error - elif instance is None: + if instance is None: return OpenRosaResponseBadRequest(_("Unable to create submission.")) audit = {"xform": instance.xform.id_string} @@ -369,13 +376,14 @@ def submission(request, username=None): # pylint: disable=R0911,R0912 except IOError as e: if _bad_request(e): return OpenRosaResponseBadRequest(_("File transfer interruption.")) - else: - raise + raise finally: if xml_file_list: - [_file.close() for _file in xml_file_list] # pylint: disable=W0106 + for _file in xml_file_list: + _file.close() if media_files: - [_file.close() for _file in media_files] # pylint: disable=W0106 + for _file in media_files: + _file.close() def download_xform(request, username, id_string): @@ -448,15 +456,13 @@ def download_xlsform(request, username, id_string): return response - else: - messages.add_message( - request, - messages.WARNING, - _("No XLS file for your form " "%(id)s") - % {"id": id_string}, - ) + messages.add_message( + request, + messages.WARNING, + _("No XLS file for your form " "%(id)s") % {"id": id_string}, + ) - return HttpResponseRedirect("/%s" % username) + return HttpResponseRedirect(f"/{username}") def download_jsonform(request, username, id_string): @@ -480,7 +486,7 @@ def download_jsonform(request, username, id_string): response = response_with_mimetype_and_name("json", id_string, show_date=False) if "callback" in request.GET and request.GET.get("callback") != "": callback = request.GET.get("callback") - response.content = "%s(%s)" % (callback, xform.json) + response.content = f"{callback}({xform.json})" else: add_cors_headers(response) response.content = xform.json @@ -540,7 +546,7 @@ def toggle_downloadable(request, username, id_string): audit, request, ) - return HttpResponseRedirect("/%s" % username) + return HttpResponseRedirect(f"/{username}") def enter_data(request, username, id_string): @@ -576,12 +582,12 @@ def enter_data(request, username, id_string): data["form_view"] = True data["message"] = { "type": "alert-error", - "text": "Enketo error, reason: %s" % e, + "text": f"Enketo error, reason: {e}", } messages.add_message( request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, + _(f"Enketo error: enketo replied {e}"), fail_silently=True, ) return render(request, "profile.html", data) @@ -608,7 +614,7 @@ def edit_data(request, username, id_string, data_id): reverse("form-show", kwargs={"username": username, "id_string": id_string}) ) - url = "%sdata/edit_url" % settings.ENKETO_URL + url = f"{settings.ENKETO_URL}data/edit_url" # see commit 220f2dad0e for tmp file creation injected_xml = inject_instanceid(instance.xml, instance.uuid) return_url = request.build_absolute_uri( @@ -616,7 +622,7 @@ def edit_data(request, username, id_string, data_id): "submission-instance", kwargs={"username": username, "id_string": id_string} ) + "#/" - + text(instance.id) + + str(instance.id) ) form_url = get_form_url(request, username, settings.ENKETO_PROTOCOL) @@ -631,12 +637,12 @@ def edit_data(request, username, id_string, data_id): except EnketoError as e: context.message = { "type": "alert-error", - "text": "Enketo error, reason: %s" % e, + "text": f"Enketo error, reason: {e}", } messages.add_message( request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, + _(f"Enketo error: enketo replied {e}"), fail_silently=True, ) else: @@ -727,7 +733,7 @@ def view_download_submission(request, username): if not has_permission(xform, form_user, request, xform.shared_data): return HttpResponseForbidden("Not shared.") submission_xml_root_node = instance.get_root_node() - submission_xml_root_node.setAttribute("instanceID", "uuid:%s" % instance.uuid) + submission_xml_root_node.setAttribute("instanceID", f"uuid:{instance.uuid}") submission_xml_root_node.setAttribute( "submissionDate", instance.date_created.isoformat() ) @@ -755,15 +761,12 @@ def form_upload(request, username): return authenticator.build_challenge_response() if form_user != request.user: return HttpResponseForbidden( - _( - "Not allowed to upload form[s] to %(user)s account." - % {"user": form_user} - ) + _(f"Not allowed to upload form[s] to {form_user} account.") ) if request.method == "HEAD": response = OpenRosaResponse(status=204) response["Location"] = request.build_absolute_uri().replace( - request.get_full_path(), "/%s/formUpload" % form_user.username + request.get_full_path(), f"/{form_user.username}/formUpload" ) return response xform_def = request.FILES.get("form_def_file", None) @@ -773,11 +776,11 @@ def form_upload(request, username): xform = publish_form(do_form_upload.publish_xform) status = 201 if isinstance(xform, XForm): - content = _("%s successfully published." % xform.id_string) + content = _(f"{xform.id_string} successfully published.") else: content = xform["text"] if isinstance(content, Exception): - content = content + content = str(content) status = 500 else: status = 400 From 35c4a1b97f5fce84a7feb3d9bc28239bfbd55d15 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 5 May 2022 23:56:00 +0300 Subject: [PATCH 139/234] exceptions.py: cleanup --- onadata/libs/exceptions.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/onadata/libs/exceptions.py b/onadata/libs/exceptions.py index c3b7103b4b..2547a421ae 100644 --- a/onadata/libs/exceptions.py +++ b/onadata/libs/exceptions.py @@ -1,35 +1,36 @@ +# -*- coding: utf-8 -*- +"""Custom Expecting classes.""" from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import APIException class EnketoError(Exception): + """Enketo specigic exceptions""" - default_message = _("There was a problem with your submissionor" - " form. Please contact support.") + default_message = _( + "There was a problem with your submission or form. Please contact support." + ) def __init__(self, message=None): - if message is None: - self.message = self.default_message - else: - self.message = message - - def __str__(self): - return "{}".format(self.message) + self.message = message if message is not None else self.default_message + super().__init__(self.message) class NoRecordsFoundError(Exception): - pass + """Raise for when no records are found.""" class NoRecordsPermission(Exception): - pass + """Raise when no permissions to access records.""" class J2XException(Exception): - pass + """Raise for json-to-xls exceptions on external exports.""" class ServiceUnavailable(APIException): + """Custom service unavailable exception.""" + status_code = 503 - default_detail = 'Service temporarily unavailable, try again later.' + default_detail = "Service temporarily unavailable, try again later." From a59cece2ca967cadccd98ec6a8c18d62b672fb94 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 6 May 2022 00:04:50 +0300 Subject: [PATCH 140/234] set_media_file_hash.py: cleanup --- .../main/management/commands/set_media_file_hash.py | 12 ++++++++---- onadata/apps/main/models/meta_data.py | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/onadata/apps/main/management/commands/set_media_file_hash.py b/onadata/apps/main/management/commands/set_media_file_hash.py index b9fa0535e9..7b1d85077c 100644 --- a/onadata/apps/main/management/commands/set_media_file_hash.py +++ b/onadata/apps/main/management/commands/set_media_file_hash.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +"""set_media_file_hash command - (re)apply the hash of all media files.""" from django.core.management.base import BaseCommand from django.utils.translation import gettext_lazy @@ -6,12 +8,14 @@ class Command(BaseCommand): - help = gettext_lazy("Set media file_hash for all existing media files") + """Set media file_hash for all existing media files""" - option_list = BaseCommand.option_list + help = gettext_lazy("Set media file_hash for all existing media files") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): - for media in queryset_iterator(MetaData.objects.exclude(data_file='')): + """Set media file_hash for all existing media files""" + for media in queryset_iterator(MetaData.objects.exclude(data_file="")): if media.data_file: - media.file_hash = media._set_hash() + media.file_hash = media.set_hash() media.save() diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index 5f2dbd1aa0..e94094b2e8 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -217,6 +217,12 @@ def hash(self): return self._set_hash() + def set_hash(self): + """ + Returns the md5 hash of the metadata file. + """ + return self._set_hash() + def _set_hash(self): """ Returns the md5 hash of the metadata file. From 0bef0782002145bcad59772477be8d308f81b826 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 6 May 2022 00:46:59 +0300 Subject: [PATCH 141/234] Check for None in overwrite params --- onadata/apps/api/viewsets/xform_viewset.py | 2 +- onadata/libs/exceptions.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 8f4839a59e..646bc5c22f 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -696,7 +696,7 @@ def data_import(self, request, *args, **kwargs): if xls_file and xls_file.name.split(".")[-1] in XLS_EXTENSIONS: csv_file = submission_xls_to_csv(xls_file) overwrite = request.query_params.get("overwrite") - overwrite = overwrite.lower() == "true" + overwrite = overwrite is not None and overwrite.lower() == "true" size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD try: csv_size = csv_file.size diff --git a/onadata/libs/exceptions.py b/onadata/libs/exceptions.py index 2547a421ae..d41c22f8de 100644 --- a/onadata/libs/exceptions.py +++ b/onadata/libs/exceptions.py @@ -29,6 +29,7 @@ class J2XException(Exception): """Raise for json-to-xls exceptions on external exports.""" +# pylint: disable=too-few-public-methods class ServiceUnavailable(APIException): """Custom service unavailable exception.""" From 51a7de559d92facc1620ee1a0e76365aba1e29a9 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 6 May 2022 23:57:37 +0300 Subject: [PATCH 142/234] create_image_thumbnails.py: cleanup --- .../commands/create_image_thumbnails.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/onadata/apps/logger/management/commands/create_image_thumbnails.py b/onadata/apps/logger/management/commands/create_image_thumbnails.py index 2b8a76e3be..5c77b21d8b 100644 --- a/onadata/apps/logger/management/commands/create_image_thumbnails.py +++ b/onadata/apps/logger/management/commands/create_image_thumbnails.py @@ -1,6 +1,10 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +create_image_thumbnails - creates thumbnails for all form images and stores them. +""" from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext as _ @@ -16,7 +20,9 @@ class Command(BaseCommand): - help = gettext_lazy("Creates thumbnails for " "all form images and stores them") + """Creates thumbnails for all form images and stores them""" + + help = gettext_lazy("Creates thumbnails for all form images and stores them") def add_arguments(self, parser): parser.add_argument( @@ -32,29 +38,28 @@ def add_arguments(self, parser): help=gettext_lazy("regenerate thumbnails if they exist."), ) + # pylint: disable=too-many-branches,too-many-locals def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( "instance", "instance__xform" ) + # pylint: disable=invalid-name + User = get_user_model() if options.get("username"): username = options.get("username") try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError( - "Error: username %(username)s does not exist" - % {"username": username} - ) + except User.DoesNotExist as e: + raise CommandError(f"Error: username {username} does not exist") from e attachments_qs = attachments_qs.filter(instance__user=user) if options.get("id_string"): id_string = options.get("id_string") try: xform = XForm.objects.get(id_string=id_string) - except XForm.DoesNotExist: + except XForm.DoesNotExist as e: raise CommandError( - "Error: Form with id_string %(id_string)s does not exist" - % {"id_string": id_string} - ) + f"Error: Form with id_string {id_string} does not exist" + ) from e attachments_qs = attachments_qs.filter(instance__xform=xform) fs = get_storage_class("django.core.files.storage.FileSystemStorage")() for att in queryset_iterator(attachments_qs): @@ -72,17 +77,10 @@ def handle(self, *args, **options): resize(filename, att.extension) else: resize_local_env(filename, att.extension) - path = get_path(filename, "%s" % THUMB_CONF["small"]["suffix"]) + path = get_path(filename, f'{THUMB_CONF["small"]["suffix"]}') if default_storage.exists(path): - self.stdout.write( - _("Thumbnails created for %(file)s") % {"file": filename} - ) + self.stdout.write(_(f"Thumbnails created for {filename}")) else: - self.stdout.write( - _("Problem with the file %(file)s") % {"file": filename} - ) - except (IOError, OSError) as e: - self.stderr.write( - _("Error on %(filename)s: %(error)s") - % {"filename": filename, "error": e} - ) + self.stdout.write(_(f"Problem with the file {filename}")) + except (IOError, OSError) as error: + self.stderr.write(_(f"Error on {filename}: {error}")) From 660f96278ffc1cac52d666d6c21a84065e872778 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 00:12:42 +0300 Subject: [PATCH 143/234] admin.py: cleanup --- onadata/apps/api/admin.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/onadata/apps/api/admin.py b/onadata/apps/api/admin.py index 1cce64406f..497104fd20 100644 --- a/onadata/apps/api/admin.py +++ b/onadata/apps/api/admin.py @@ -5,8 +5,9 @@ from onadata.apps.api.models import Team, OrganizationProfile, TempToken -class TeamAdmin(admin.ModelAdmin): - """Filter by request.user unless is_superuser.""" +# pylint: disable=too-few-public-methods +class FilterSuperuserMixin: + """Filter by request user and give full access to superuser.""" def get_queryset(self, request): """Filter by request.user unless is_superuser.""" @@ -16,32 +17,22 @@ def get_queryset(self, request): return queryset.filter(user=request.user) +class TeamAdmin(FilterSuperuserMixin, admin.ModelAdmin): + """Filter by request.user unless is_superuser.""" + + admin.site.register(Team, TeamAdmin) -class OrganizationProfileAdmin(admin.ModelAdmin): +class OrganizationProfileAdmin(FilterSuperuserMixin, admin.ModelAdmin): """Filter by request.user unless is_superuser.""" - def get_queryset(self, request): - """Filter by request.user unless is_superuser.""" - queryset = super().get_queryset(request) - if request.user.is_superuser: - return queryset - return queryset.filter(user=request.user) - admin.site.register(OrganizationProfile, OrganizationProfileAdmin) -class TempTokenProfileAdmin(admin.ModelAdmin): +class TempTokenProfileAdmin(FilterSuperuserMixin, admin.ModelAdmin): """Filter by request.user unless is_superuser.""" - def get_queryset(self, request): - """Filter by request.user unless is_superuser.""" - queryset = super().get_queryset(request) - if request.user.is_superuser: - return queryset - return queryset.filter(user=request.user) - admin.site.register(TempToken, TempTokenProfileAdmin) From f76fab491f7ef7a40ce5cc42c07c9b6e17c43c3e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 05:38:52 +0300 Subject: [PATCH 144/234] viewer/views.py: cleanup --- onadata/apps/viewer/views.py | 153 +++++++++++++++-------------------- 1 file changed, 65 insertions(+), 88 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 438e10ca9b..722d2b4e0b 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -1,20 +1,17 @@ # -*- coding: utf-8 -*- +# pylint: disable=too-many-lines """ data views. """ -import json import os from datetime import datetime from tempfile import NamedTemporaryFile from time import strftime, strptime from wsgiref.util import FileWrapper -import pytz -import requests -from dict2xml import dict2xml from django.conf import settings +from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage, get_storage_class from django.http import ( HttpResponse, @@ -22,12 +19,17 @@ HttpResponseForbidden, HttpResponseNotFound, HttpResponseRedirect, + JsonResponse, ) from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST + +import pytz +import requests +from dict2xml import dict2xml from dpath import util as dpath_util from oauth2client import client as google_client from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage @@ -46,6 +48,7 @@ from onadata.apps.viewer.xls_writer import XlsWriter from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.chart_tools import build_chart_data +from onadata.libs.utils.common_tools import get_uuid from onadata.libs.utils.export_tools import ( DEFAULT_GROUP_DELIMITER, generate_export, @@ -71,7 +74,9 @@ export_def_from_filename, get_form, ) -from onadata.libs.utils.common_tools import get_uuid + +# pylint: disable=invalid-name +User = get_user_model() def _get_start_end_submission_time(request): @@ -114,7 +119,7 @@ def instances_for_export(data_dictionary, start=None, end=None): """ Returns Instance submission queryset filtered by start and end dates. """ - kwargs = dict() + kwargs = {} if start: kwargs["date_created__gte"] = start if end: @@ -207,7 +212,7 @@ def map_view(request, username, id_string, template="map.html"): Actions.FORM_MAP_VIEWED, request.user, owner, - _("Requested map on '%(id_string)s'.") % {"id_string": xform.id_string}, + _(f"Requested map on '{xform.id_string}'."), audit, request, ) @@ -257,9 +262,9 @@ def geopoint_xpaths(username, id_string): "thank_you_submission", kwargs={"username": username, "id_string": id_string} ) if settings.DEBUG: - openrosa_url = "https://dev.formhub.org/{}".format(username) + openrosa_url = f"https://dev.formhub.org/{username}" else: - openrosa_url = request.build_absolute_uri("/{}".format(username)) + openrosa_url = request.build_absolute_uri(f"/{username}") payload = { "return_url": return_url, "form_id": id_string, @@ -275,10 +280,11 @@ def geopoint_xpaths(username, id_string): verify=getattr(settings, "VERIFY_SSL", True), ) + # pylint: disable=http-response-with-content-type-json return HttpResponse(response.text, content_type="application/json") -# pylint: disable=W0613 +# pylint: disable=unused-argument def thank_you_submission(request, username, id_string): """ Thank you view after successful submission. @@ -286,13 +292,14 @@ def thank_you_submission(request, username, id_string): return HttpResponse("Thank You") -# pylint: disable=R0914 +# pylint: disable=too-many-locals def data_export(request, username, id_string, export_type): """ Data export view. """ owner = get_object_or_404(User, username__iexact=username) xform = get_form({"user": owner, "id_string__iexact": id_string}) + id_string = xform.id_string helper_auth_helper(request) if not has_permission(xform, owner, request): @@ -325,14 +332,15 @@ def data_export(request, username, id_string, export_type): start, end = _get_start_end_submission_time(request) options.update({"start": start, "end": end}) + # pylint: disable=broad-except try: export = generate_export(export_type, xform, None, options) + export_type = export_type.upper() audit_log( Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") - % {"id_string": xform.id_string, "export_type": export_type.upper()}, + _(f"Created {export_type} export on '{id_string}'."), audit, request, ) @@ -342,14 +350,14 @@ def data_export(request, username, id_string, export_type): return HttpResponseBadRequest(str(e)) else: export = newest_export_for(xform, export_type, options) + export_type = export_type.upper() # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export on '%(id_string)s'.") - % {"id_string": xform.id_string, "export_type": export_type.upper()}, + _(f"Downloaded {export_type} export on '{id_string}'."), audit, request, ) @@ -357,7 +365,7 @@ def data_export(request, username, id_string, export_type): if not export.filename and not export.error_message: # tends to happen when using newset_export_for. return HttpResponseNotFound("File does not exist!") - elif not export.filename and export.error_message: + if not export.filename and export.error_message: return HttpResponseBadRequest(str(export.error_message)) # get extension from file_path, exporter could modify to @@ -377,7 +385,7 @@ def data_export(request, username, id_string, export_type): return response -# pylint: disable=R0914 +# pylint: disable=too-many-locals @login_required @require_POST def create_export(request, username, id_string, export_type): @@ -408,9 +416,7 @@ def create_export(request, username, id_string, export_type): # export options group_delimiter = request.POST.get("options[group_delimiter]", "/") if group_delimiter not in [".", "/"]: - return HttpResponseBadRequest( - _("%s is not a valid delimiter" % group_delimiter) - ) + return HttpResponseBadRequest(_(f"{group_delimiter} is not a valid delimiter")) # default is True, so when dont_.. is yes # split_select_multiples becomes False @@ -439,18 +445,16 @@ def create_export(request, username, id_string, export_type): try: create_async_export(xform, export_type, query, force_xlsx, options) except ExportTypeError: - return HttpResponseBadRequest(_("%s is not a valid export type" % export_type)) + return HttpResponseBadRequest(_(f"{export_type} is not a valid export type")) else: audit = {"xform": xform.id_string, "export_type": export_type} + export_type = export_type.upper() + id_string = xform.id_string audit_log( Actions.EXPORT_CREATED, request.user, owner, - _("Created %(export_type)s export on '%(id_string)s'.") - % { - "export_type": export_type.upper(), - "id_string": xform.id_string, - }, + _(f"Created {export_type} export on '{id_string}'."), audit, request, ) @@ -485,7 +489,7 @@ def export_list(request, username, id_string, export_type): if export_type not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( - _('Export type "%s" is not supported.' % export_type) + _(f'Export type "{export_type}" is not supported.') ) if export_type == Export.GOOGLE_SHEETS_EXPORT: @@ -521,9 +525,9 @@ def export_list(request, username, id_string, export_type): create_async_export( xform, export_type, query=None, force_xlsx=True, options=options ) - except Export.ExportTypeError: + except ExportTypeError: return HttpResponseBadRequest( - _("%s is not a valid export type" % export_type) + _(f"{export_type} is not a valid export type") ) metadata_qs = MetaData.objects.filter( @@ -593,11 +597,11 @@ def export_progress(request, username, id_string, export_type): ): status["url"] = None # mark as complete if it either failed or succeeded but NOT pending - if export.status == Export.SUCCESSFUL or export.status == Export.FAILED: + if export.status in [Export.SUCCESSFUL, Export.FAILED]: status["complete"] = True statuses.append(status) - return HttpResponse(json.dumps(statuses), content_type="application/json") + return JsonResponse(statuses, safe=False) def export_download(request, username, id_string, export_type, filename): @@ -614,25 +618,24 @@ def export_download(request, username, id_string, export_type, filename): # find the export entry in the db export = get_object_or_404(Export, xform=xform, filename=filename) - if ( - export_type == Export.GOOGLE_SHEETS_EXPORT - or export_type == Export.EXTERNAL_EXPORT - ) and export.export_url is not None: + is_external_export = export_type in [ + Export.GOOGLE_SHEETS_EXPORT, + Export.EXTERNAL_EXPORT, + ] + if is_external_export and export.export_url is not None: return HttpResponseRedirect(export.export_url) ext, mime_type = export_def_from_filename(export.filename) + export_type = export.export_type.upper() + filename = export.filename + id_string = xform.id_string audit = {"xform": xform.id_string, "export_type": export.export_type} audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded %(export_type)s export '%(filename)s' " "on '%(id_string)s'.") - % { - "export_type": export.export_type.upper(), - "filename": export.filename, - "id_string": xform.id_string, - }, + _(f"Downloaded {export_type} export '{filename}' on '{id_string}'."), audit, request, ) @@ -669,19 +672,17 @@ def delete_export(request, username, id_string, export_type): # find the export entry in the db export = get_object_or_404(Export, id=export_id) - + export_type = export.export_type.upper() export.delete() + + filename = export.filename + id_string = xform.id_string audit = {"xform": xform.id_string, "export_type": export.export_type} audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Deleted %(export_type)s export '%(filename)s'" " on '%(id_string)s'.") - % { - "export_type": export.export_type.upper(), - "filename": export.filename, - "id_string": xform.id_string, - }, + _(f"Deleted {export_type} export '{filename}' on '{id_string}'."), audit, request, ) @@ -720,10 +721,7 @@ def zip_export(request, username, id_string): Actions.EXPORT_CREATED, request.user, owner, - _("Created ZIP export on '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Created ZIP export on '{xform.id_string}'."), audit, request, ) @@ -732,10 +730,7 @@ def zip_export(request, username, id_string): Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded ZIP export on '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Downloaded ZIP export on '{xform.id_string}'."), audit, request, ) @@ -776,10 +771,7 @@ def kml_export(request, username, id_string): Actions.EXPORT_CREATED, request.user, owner, - _("Created KML export on '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Created KML export on '{xform.id_string}'."), audit, request, ) @@ -788,10 +780,7 @@ def kml_export(request, username, id_string): Actions.EXPORT_DOWNLOADED, request.user, owner, - _("Downloaded KML export on '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Downloaded KML export on '{xform.id_string}'."), audit, request, ) @@ -832,11 +821,11 @@ def google_xls_export(request, username, id_string): return data_dictionary xls_writer = XlsWriter() - tmp = NamedTemporaryFile(delete=False) - xls_writer.set_file(tmp) - xls_writer.set_data_dictionary(data_dictionary) - temp_file = xls_writer.save_workbook_to_file() - temp_file.close() + with NamedTemporaryFile(delete=False) as tmp: + xls_writer.set_file(tmp) + xls_writer.set_data_dictionary(data_dictionary) + temp_file = xls_writer.save_workbook_to_file() + temp_file.close() url = None os.unlink(tmp.name) audit = {"xform": xform.id_string, "export_type": "google"} @@ -844,10 +833,7 @@ def google_xls_export(request, username, id_string): Actions.EXPORT_CREATED, request.user, owner, - _("Created Google Docs export on '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Created Google Docs export on '{xform.id_string}'."), audit, request, ) @@ -872,10 +858,7 @@ def data_view(request, username, id_string): Actions.FORM_DATA_VIEWED, request.user, owner, - _("Requested data view for '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Requested data view for '{xform.id_string}'."), audit, request, ) @@ -919,8 +902,7 @@ def instance(request, username, id_string): """ Data view for browsing submissions one at a time. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( + xform, _is_owner, can_edit, can_view = get_xform_and_perms( username, id_string, request ) # no access @@ -938,10 +920,7 @@ def instance(request, username, id_string): Actions.FORM_DATA_VIEWED, request.user, xform.user, - _("Requested instance view for '%(id_string)s'.") - % { - "id_string": xform.id_string, - }, + _(f"Requested instance view for '{xform.id_string}'."), audit, request, ) @@ -962,8 +941,7 @@ def charts(request, username, id_string): """ Charts view. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( + xform, _is_owner, _can_edit, can_view = get_xform_and_perms( username, id_string, request ) @@ -1003,8 +981,7 @@ def stats_tables(request, username, id_string): """ Stats view. """ - # pylint: disable=W0612 - xform, is_owner, can_edit, can_view = get_xform_and_perms( + xform, _is_owner, _can_edit, can_view = get_xform_and_perms( username, id_string, request ) # no access From b4af3d98df8bf47c1f7bb65ca66868199675ce2c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 05:47:03 +0300 Subject: [PATCH 145/234] restore_backup.py: cleanup --- .../management/commands/restore_backup.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/onadata/apps/logger/management/commands/restore_backup.py b/onadata/apps/logger/management/commands/restore_backup.py index 4d03c0b37e..10d91b2918 100644 --- a/onadata/apps/logger/management/commands/restore_backup.py +++ b/onadata/apps/logger/management/commands/restore_backup.py @@ -1,7 +1,11 @@ +# -*- coding: utf-8 -*- +""" +restore_backup command - Restore a zip backup of a form and all its submissions +""" import os import sys -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext_lazy, gettext as _ @@ -9,21 +13,27 @@ class Command(BaseCommand): + """ + restore_backup command - Restore a zip backup of a form and all its submissions + """ + args = "username input_file" - help = gettext_lazy("Restore a zip backup of a form and all its" " submissions") + help = gettext_lazy("Restore a zip backup of a form and all its submissions") def handle(self, *args, **options): + # pylint: disable=invalid-name + User = get_user_model() try: username = args[0] except IndexError: raise CommandError( - _("You must provide the username to publish the" " form to.") + _("You must provide the username to publish the form to.") ) # make sure user exists try: User.objects.get(username=username) except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + raise CommandError(_(f"The user '{username}' does not exist.")) try: input_file = args[1] @@ -33,6 +43,4 @@ def handle(self, *args, **options): input_file = os.path.realpath(input_file) num_instances, num_restored = restore_backup_from_zip(input_file, username) - sys.stdout.write( - "Restored %d of %d submissions\n" % (num_restored, num_instances) - ) + sys.stdout.write(f"Restored {num_restored} of {num_instances } submissions\n") From 891fc5166934b215f3087d25c61c15b5d270c0b5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 06:45:01 +0300 Subject: [PATCH 146/234] project_serializer.py: cleanup --- .../libs/serializers/project_serializer.py | 82 +++++++++++-------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index e0b79a88c3..5f93a91e07 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -5,7 +5,7 @@ from six import itervalues from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.cache import cache from django.db.utils import IntegrityError from django.utils.translation import gettext as _ @@ -14,14 +14,14 @@ from onadata.apps.api.models import OrganizationProfile from onadata.apps.api.tools import ( - get_organization_members_team, get_or_create_organization_owners_team, + get_organization_members_team, ) from onadata.apps.logger.models import Project, XForm from onadata.libs.permissions import ( + ManagerRole, OwnerRole, ReadOnlyRole, - ManagerRole, get_role, is_organization, ) @@ -33,15 +33,18 @@ PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, PROJECT_LINKED_DATAVIEWS, - PROJ_OWNER_CACHE, safe_delete, ) from onadata.libs.utils.decorators import check_obj +# pylint: disable=invalid-name +User = get_user_model() + def get_project_xforms(project): """ @@ -62,7 +65,8 @@ def get_last_submission_date(project): :param project: The project to find the last submission date for. """ - last_submission_date = cache.get("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk)) + cache_key = f"{PROJ_SUB_DATE_CACHE}{project.pk}" + last_submission_date = cache.get(cache_key) if last_submission_date: return last_submission_date xforms = get_project_xforms(project) @@ -72,7 +76,7 @@ def get_last_submission_date(project): dates.sort(reverse=True) last_submission_date = dates[0] if dates else None - cache.set("{}{}".format(PROJ_SUB_DATE_CACHE, project.pk), last_submission_date) + cache.set(cache_key, last_submission_date) return last_submission_date @@ -83,12 +87,13 @@ def get_num_datasets(project): :param project: The project to find datasets for. """ - count = cache.get("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk)) + project_cache_key = f"{PROJ_NUM_DATASET_CACHE}{project.pk}" + count = cache.get(project_cache_key) if count: return count count = len(get_project_xforms(project)) - cache.set("{}{}".format(PROJ_NUM_DATASET_CACHE, project.pk), count) + cache.set(project_cache_key, count) return count @@ -113,7 +118,8 @@ def get_teams(project): """ Return the teams with access to the project. """ - teams_users = cache.get("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk)) + project_team_cache_key = f"{PROJ_TEAM_USERS_CACHE}{project.pk}" + teams_users = cache.get(project_team_cache_key) if teams_users: return teams_users @@ -129,7 +135,7 @@ def get_teams(project): {"name": team.name, "role": get_role(perms, project), "users": users} ) - cache.set("{}{}".format(PROJ_TEAM_USERS_CACHE, project.pk), teams_users) + cache.set(project_team_cache_key, teams_users) return teams_users @@ -138,8 +144,9 @@ def get_users(project, context, all_perms=True): """ Return a list of users and organizations that have access to the project. """ + project_permissions_cache_key = f"{PROJ_PERM_CACHE}{project.pk}" if all_perms: - users = cache.get("{}{}".format(PROJ_PERM_CACHE, project.pk)) + users = cache.get(project_permissions_cache_key) if users: return users @@ -187,7 +194,7 @@ def get_users(project, context, all_perms=True): results = list(itervalues(data)) if all_perms: - cache.set("{}{}".format(PROJ_PERM_CACHE, project.pk), results) + cache.set(project_permissions_cache_key, results) return results @@ -197,6 +204,7 @@ def set_owners_permission(user, project): OwnerRole.add(user, project) +# pylint: disable=too-few-public-methods class BaseProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ BaseProjectXFormSerializer class. @@ -205,11 +213,13 @@ class BaseProjectXFormSerializer(serializers.HyperlinkedModelSerializer): formid = serializers.ReadOnlyField(source="id") name = serializers.ReadOnlyField(source="title") + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm fields = ("name", "formid", "id_string", "is_merged_dataset") +# pylint: disable=too-few-public-methods class ProjectXFormSerializer(serializers.HyperlinkedModelSerializer): """ ProjectXFormSerializer class - to return project xform info. @@ -320,7 +330,8 @@ def get_forms(self, obj): """ Return list of xforms in the project. """ - forms = cache.get("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk)) + project_forms_cache_key = f"{PROJ_BASE_FORMS_CACHE}{obj.pk}" + forms = cache.get(project_forms_cache_key) if forms: return forms @@ -330,7 +341,7 @@ def get_forms(self, obj): xforms, context={"request": request}, many=True ) forms = list(serializer.data) - cache.set("{}{}".format(PROJ_BASE_FORMS_CACHE, obj.pk), forms) + cache.set(project_forms_cache_key, forms) return forms @@ -404,6 +415,8 @@ class Meta: exclude = ("shared", "user_stars", "deleted_by", "organization") def validate(self, attrs): + """Validate the project name does not exist and the user has the permissions to + create a project in the organization.""" name = attrs.get("name") organization = attrs.get("organization") if not self.instance and organization: @@ -412,7 +425,7 @@ def validate(self, attrs): ) if project_w_same_name: raise serializers.ValidationError( - {"name": _("Project {} already exists.".format(name))} + {"name": _(f"Project {name} already exists.")} ) else: organization = organization or self.instance.organization @@ -428,8 +441,7 @@ def validate(self, attrs): { "owner": _( "You do not have permission to create a project " - "in the organization %(organization)s." - % {"organization": organization} + f"in the organization {organization}." ) } ) @@ -452,8 +464,8 @@ def validate_metadata(self, value): # pylint: disable=no-self-use msg = serializers.ValidationError(_("Invaid value for metadata")) try: json_val = JsonField.to_json(value) - except ValueError: - raise serializers.ValidationError(msg) + except ValueError as e: + raise serializers.ValidationError(msg) from e else: if json_val is None: raise serializers.ValidationError(msg) @@ -462,7 +474,7 @@ def validate_metadata(self, value): # pylint: disable=no-self-use def update(self, instance, validated_data): metadata = JsonField.to_json(validated_data.get("metadata")) if metadata is None: - metadata = dict() + metadata = {} owner = validated_data.get("organization") if self.partial and metadata: @@ -490,13 +502,15 @@ def update(self, instance, validated_data): members = members.exclude(username=owner.username) # Add permissions to all users in Owners and Members team - [OwnerRole.add(owner, instance) for owner in owners] - [ReadOnlyRole.add(member, instance) for member in members] + for owner in owners: + OwnerRole.add(owner, instance) + for member in members: + ReadOnlyRole.add(member, instance) # clear cache - safe_delete("{}{}".format(PROJ_PERM_CACHE, instance.pk)) + safe_delete(f"{PROJ_PERM_CACHE}{instance.pk}") - project = super(ProjectSerializer, self).update(instance, validated_data) + project = super().update(instance, validated_data) project.xform_set.exclude(shared=project.shared).update( shared=project.shared, shared_data=project.shared @@ -513,23 +527,23 @@ def update(self, instance, validated_data): }, ) def create(self, validated_data): - metadata = validated_data.get("metadata", dict()) + metadata = validated_data.get("metadata", {}) if metadata is None: - metadata = dict() + metadata = {} created_by = self.context["request"].user try: - project = Project.objects.create( # pylint: disable=E1101 + project = Project.objects.create( # pylint: disable=no-member name=validated_data.get("name"), organization=validated_data.get("organization"), created_by=created_by, shared=validated_data.get("shared", False), metadata=metadata, ) - except IntegrityError: + except IntegrityError as e: raise serializers.ValidationError( "The fields name, organization must make a unique set." - ) + ) from e else: project.xform_set.exclude(shared=project.shared).update( shared=project.shared, shared_data=project.shared @@ -552,7 +566,8 @@ def get_forms(self, obj): # pylint: disable=no-self-use """ Return list of xforms in the project. """ - forms = cache.get("{}{}".format(PROJ_FORMS_CACHE, obj.pk)) + project_forms_cache_key = f"{PROJ_FORMS_CACHE}{obj.pk}" + forms = cache.get(project_forms_cache_key) if forms: return forms xforms = get_project_xforms(obj) @@ -561,7 +576,7 @@ def get_forms(self, obj): # pylint: disable=no-self-use xforms, context={"request": request}, many=True ) forms = list(serializer.data) - cache.set("{}{}".format(PROJ_FORMS_CACHE, obj.pk), forms) + cache.set(project_forms_cache_key, forms) return forms @@ -594,7 +609,8 @@ def get_data_views(self, obj): """ Return a list of filtered datasets. """ - data_views = cache.get("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk)) + project_dataview_cache_key = f"{PROJECT_LINKED_DATAVIEWS}{obj.pk}" + data_views = cache.get(project_dataview_cache_key) if data_views: return data_views @@ -609,6 +625,6 @@ def get_data_views(self, obj): ) data_views = list(serializer.data) - cache.set("{}{}".format(PROJECT_LINKED_DATAVIEWS, obj.pk), data_views) + cache.set(project_dataview_cache_key, data_views) return data_views From 60a7793c31dea800e89c96fbeedd7f5f8668efdc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 07:03:16 +0300 Subject: [PATCH 147/234] parsed_instance.py: cleanup --- onadata/apps/viewer/models/parsed_instance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 1abe10543e..3882904d35 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -304,6 +304,7 @@ def query_data( count=None, json_only: bool = True, ): + """Query the submissions table and returns the results.""" sql, params, records = get_sql_with_params( xform, From 9b7094b47031c0eb47ba77251e2b9cc229d905dc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 11:27:57 +0300 Subject: [PATCH 148/234] xform_serializer.p: cleanup --- onadata/libs/serializers/xform_serializer.py | 43 +++++++++++++------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 7f60420d61..41dac1a253 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -2,12 +2,13 @@ """ XForm model serialization. """ +import hashlib import logging import os -from hashlib import md5 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.serialization import load_pem_public_key +from six import itervalues +from six.moves.urllib.parse import urlparse + from django.conf import settings from django.contrib.auth import get_user_model from django.core.cache import cache @@ -15,14 +16,14 @@ from django.core.validators import URLValidator from django.db.models import Count from django.utils.translation import gettext as _ -from six.moves.urllib.parse import urlparse -from six import itervalues + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.serialization import load_pem_public_key from rest_framework import serializers from rest_framework.reverse import reverse -from onadata.apps.logger.models import DataView, Instance, XForm +from onadata.apps.logger.models import DataView, Instance, XForm, XFormVersion from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.logger.models import XFormVersion from onadata.libs.exceptions import EnketoError from onadata.libs.permissions import get_role, is_organization from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer @@ -30,19 +31,18 @@ from onadata.libs.serializers.tag_list_serializer import TagListSerializer from onadata.libs.utils.cache_tools import ( ENKETO_PREVIEW_URL_CACHE, - ENKETO_URL_CACHE, ENKETO_SINGLE_SUBMIT_URL_CACHE, + ENKETO_URL_CACHE, + XFORM_COUNT, + XFORM_DATA_VERSIONS, XFORM_LINKED_DATAVIEWS, XFORM_METADATA_CACHE, XFORM_PERMISSIONS_CACHE, - XFORM_DATA_VERSIONS, - XFORM_COUNT, ) from onadata.libs.utils.common_tags import GROUP_DELIMETER_TAG, REPEAT_INDEX_TAGS from onadata.libs.utils.decorators import check_obj from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url - SUBMISSION_RETRIEVAL_THRESHOLD = getattr( settings, "SUBMISSION_RETRIEVAL_THRESHOLD", 10000 ) @@ -129,6 +129,7 @@ def clean_public_key(value): return value +# pylint: disable=too-few-public-methods class MultiLookupIdentityField(serializers.HyperlinkedIdentityField): """ Custom HyperlinkedIdentityField that supports multiple lookup fields. @@ -285,6 +286,7 @@ def get_enketo_preview_url(self, obj): return None def get_data_views(self, obj): + """Returns a list of filtered datasets linked to the form.""" if obj: key = f"{XFORM_LINKED_DATAVIEWS}{obj.pk}" data_views = cache.get(key) @@ -343,6 +345,8 @@ def get_last_submission_time(self, obj): class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): + """XForm base serializer.""" + formid = serializers.ReadOnlyField(source="id") owner = serializers.HyperlinkedRelatedField( view_name="user-detail", @@ -377,6 +381,7 @@ class XFormBaseSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): data_views = serializers.SerializerMethodField() xls_available = serializers.SerializerMethodField() + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm read_only_fields = ( @@ -404,6 +409,10 @@ class Meta: class XFormSerializer(XFormMixin, serializers.HyperlinkedModelSerializer): + """ + XForm model serializer + """ + formid = serializers.ReadOnlyField(source="id") metadata = serializers.SerializerMethodField() owner = serializers.HyperlinkedRelatedField( @@ -547,6 +556,7 @@ def get_form_versions(self, obj): return versions +# pylint: disable=abstract-method class XFormCreateSerializer(XFormSerializer): """ XForm serializer that is only relevant during the XForm publishing process. @@ -562,6 +572,7 @@ def get_has_id_string_changed(self, obj): return obj.has_id_string_changed +# pylint: disable=abstract-method class XFormListSerializer(serializers.Serializer): """ XForm serializer for OpenRosa form list API. @@ -617,7 +628,8 @@ def get_url(self, obj): } request = self.context.get("request") try: - fmt = obj.data_value[obj.data_value.rindex(".") + 1 :] + fmt_index = obj.data_value.rindex(".") + 1 + fmt = obj.data_value[fmt_index:] except ValueError: fmt = "csv" url = reverse("xform-media", kwargs=kwargs, request=request, format=fmt.lower()) @@ -660,8 +672,10 @@ def get_hash(self, obj): xform = data_view.xform if xform and xform.last_submission_time: - md5_hash = md5( - xform.last_submission_time.isoformat().encode("utf-8") + md5_hash = hashlib.new( + "md5", + xform.last_submission_time.isoformat().encode("utf-8"), + usedforsecurity=False, ).hexdigest() hsh = f"md5:{md5_hash}" @@ -691,6 +705,7 @@ def get_filename(self, obj): return filename +# pylint: disable=too-few-public-methods class XFormVersionListSerializer(serializers.ModelSerializer): """ XFormVersion list API serializer From 55e9b66de492ec52931de91d1fc93b3f8c3e8b51 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 12:01:18 +0300 Subject: [PATCH 149/234] user_profile_serializer.py: cleanup --- .../serializers/user_profile_serializer.py | 75 +++++++++++-------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index 8af5c82b03..8172a3fb6a 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -5,36 +5,39 @@ import copy import re - from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.sites.models import Site from django.core.cache import cache from django.db import IntegrityError, transaction -from django.utils.translation import gettext as _ +from django.db.models.query import QuerySet from django.utils import timezone +from django.utils.translation import gettext as _ import six from django_digest.backend.db import update_partial_digests -from django.db.models.query import QuerySet from registration.models import RegistrationProfile from rest_framework import serializers -from onadata.apps.api.tasks import send_verification_email from onadata.apps.api.models.temp_token import TempToken +from onadata.apps.api.tasks import send_verification_email from onadata.apps.main.forms import RegistrationFormUserProfile from onadata.apps.main.models import UserProfile from onadata.libs.authentication import expired from onadata.libs.permissions import CAN_VIEW_PROFILE, is_organization from onadata.libs.serializers.fields.json_field import JsonField -from onadata.libs.utils.cache_tools import IS_ORG from onadata.libs.utils.analytics import track_object_event -from onadata.libs.utils.email import get_verification_url, get_verification_email_data +from onadata.libs.utils.cache_tools import IS_ORG +from onadata.libs.utils.email import get_verification_email_data, get_verification_url RESERVED_NAMES = RegistrationFormUserProfile.RESERVED_USERNAMES LEGAL_USERNAMES_REGEX = RegistrationFormUserProfile.legal_usernames_re +# pylint: disable=invalid-name +User = get_user_model() + + def _get_first_last_names(name, limit=30): if not isinstance(name, six.string_types): return name, name @@ -44,13 +47,14 @@ def _get_first_last_names(name, limit=30): # imposition of 30 characters on both first_name and last_name hence # ensure we only have 30 characters for either field - return name[:limit], name[limit : limit * 2] + end = limit * 2 + return name[:limit], name[limit:end] name_split = name.split() first_name = name_split[0] last_name = "" - if len(name_split) > 1: + if name_split: last_name = " ".join(name_split[1:]) return first_name, last_name @@ -133,9 +137,11 @@ class UserProfileSerializer(serializers.HyperlinkedModelSerializer): view_name="user-detail", lookup_field="username", read_only=True ) metadata = JsonField(required=False) - id = serializers.ReadOnlyField(source="user.id") # pylint: disable=C0103 + # pylint: disable=invalid-name + id = serializers.ReadOnlyField(source="user.id") joined_on = serializers.ReadOnlyField(source="user.date_joined") + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = UserProfile fields = ( @@ -162,7 +168,7 @@ class Meta: owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): - super(UserProfileSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if self.instance and hasattr(self.Meta, "owner_only_fields"): request = self.context.get("request") if ( @@ -178,19 +184,19 @@ def get_is_org(self, obj): # pylint: disable=no-self-use Returns True if it is an organization profile. """ if obj: - is_org = cache.get("{}{}".format(IS_ORG, obj.pk)) + is_org = cache.get(f"{IS_ORG}{obj.pk}") if is_org: return is_org is_org = is_organization(obj) - cache.set("{}{}".format(IS_ORG, obj.pk), is_org) + cache.set(f"{IS_ORG}{obj.pk}", is_org) return is_org def to_representation(self, instance): """ Serialize objects -> primitives. """ - ret = super(UserProfileSerializer, self).to_representation(instance) + ret = super().to_representation(instance) if "password" in ret: del ret["password"] @@ -211,6 +217,7 @@ def to_representation(self, instance): return ret def update(self, instance, validated_data): + """Update user properties.""" params = validated_data password = params.get("password1") email = params.get("email") @@ -218,7 +225,7 @@ def update(self, instance, validated_data): # Check password if email is being updated if email and not password: raise serializers.ValidationError( - _("Your password is required when updating your email " "address.") + _("Your password is required when updating your email address.") ) if password and not instance.user.check_password(password): raise serializers.ValidationError(_("Invalid password")) @@ -246,29 +253,30 @@ def update(self, instance, validated_data): # force django-digest to regenerate its stored partial digests update_partial_digests(instance.user, password) - return super(UserProfileSerializer, self).update(instance, params) + return super().update(instance, params) @track_object_event( user_field="user", properties={"name": "name", "country": "country"} ) def create(self, validated_data): + """Creates a user registration profile and account.""" params = validated_data request = self.context.get("request") metadata = {} - + username = params.get("username") site = Site.objects.get(pk=settings.SITE_ID) try: new_user = RegistrationProfile.objects.create_inactive_user( - username=params.get("username"), + username=username, password=params.get("password1"), email=params.get("email"), site=site, send_email=settings.SEND_EMAIL_ACTIVATION_API, ) - except IntegrityError: + except IntegrityError as e: raise serializers.ValidationError( - _("User account {} already exists".format(params.get("username"))) - ) + _(f"User account {username} already exists") + ) from e new_user.is_active = True new_user.first_name = params.get("first_name") new_user.last_name = params.get("last_name") @@ -293,6 +301,7 @@ def create(self, validated_data): metadata=metadata, ) profile.save() + return profile def validate_username(self, value): @@ -303,16 +312,16 @@ def validate_username(self, value): if username in RESERVED_NAMES: raise serializers.ValidationError( - _("%s is a reserved name, please choose another" % username) + _(f"{username} is a reserved name, please choose another") ) - elif not LEGAL_USERNAMES_REGEX.search(username): + if not LEGAL_USERNAMES_REGEX.search(username): raise serializers.ValidationError( _( "username may only contain alpha-numeric characters and " "underscores" ) ) - elif len(username) < 3: + if len(username) < 3: raise serializers.ValidationError( _("Username must have 3 or more characters") ) @@ -320,7 +329,7 @@ def validate_username(self, value): if self.instance: users = users.exclude(pk=self.instance.user.pk) if users.exists(): - raise serializers.ValidationError(_("%s already exists" % username)) + raise serializers.ValidationError(_(f"{username} already exists")) return username @@ -347,7 +356,7 @@ def validate_twitter(self, value): # pylint: disable=no-self-use match = re.search(r"^[A-Za-z0-9_]{1,15}$", value) if not match: raise serializers.ValidationError( - _("Invalid twitter username {}".format(value)) + _(f"Invalid twitter username {value}") ) return value @@ -403,24 +412,26 @@ class Meta: "temp_token", ) - def get_api_token(self, object): # pylint: disable=R0201,W0622 + # pylint: disable=no-self-use + def get_api_token(self, obj): """ Returns user's API Token. """ - return object.user.auth_token.key + return obj.user.auth_token.key - def get_temp_token(self, object): # pylint: disable=R0201,W0622 + # pylint: disable=no-self-use + def get_temp_token(self, obj): """ This should return a valid temp token for this user profile. """ - token, created = TempToken.objects.get_or_create(user=object.user) + token, created = TempToken.objects.get_or_create(user=obj.user) check_expired = getattr(settings, "CHECK_EXPIRED_TEMP_TOKEN", True) try: if check_expired and not created and expired(token.created): with transaction.atomic(): - TempToken.objects.get(user=object.user).delete() - token = TempToken.objects.create(user=object.user) + TempToken.objects.get(user=obj.user).delete() + token = TempToken.objects.create(user=obj.user) except IntegrityError: pass From 41218139f653d41b8a782405eb6d5a5f52ff89c7 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 12:20:06 +0300 Subject: [PATCH 150/234] organization_serializer.py: cleanup --- .../serializers/organization_serializer.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index 2d5c8d7fab..a000e611df 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -3,7 +3,7 @@ Organization Serializer """ -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.db.models.query import QuerySet from django.utils.translation import gettext as _ @@ -20,6 +20,9 @@ from onadata.libs.permissions import get_role_in_org from onadata.libs.serializers.fields.json_field import JsonField +# pylint: disable=invalid-name +User = get_user_model() + class OrganizationSerializer(serializers.HyperlinkedModelSerializer): """ @@ -46,7 +49,7 @@ class Meta: owner_only_fields = ("metadata",) def __init__(self, *args, **kwargs): - super(OrganizationSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if self.instance and hasattr(self.Meta, "owner_only_fields"): request = self.context.get("request") @@ -60,6 +63,7 @@ def __init__(self, *args, **kwargs): self.fields.pop(field) def update(self, instance, validated_data): + """Update organization profile properties.""" # update the user model if "name" in validated_data: first_name, last_name = _get_first_last_names(validated_data.get("name")) @@ -67,9 +71,10 @@ def update(self, instance, validated_data): instance.user.last_name = last_name instance.user.save() - return super(OrganizationSerializer, self).update(instance, validated_data) + return super().update(instance, validated_data) def create(self, validated_data): + """Create an organization profile.""" org = validated_data.get("user") if org: org = org.get("username") @@ -95,9 +100,9 @@ def validate_org(self, value): # pylint: disable=no-self-use if org in RegistrationFormUserProfile.RESERVED_USERNAMES: raise serializers.ValidationError( - _("%s is a reserved name, please choose another" % org) + _(f"{org} is a reserved name, please choose another") ) - elif not RegistrationFormUserProfile.legal_usernames_re.search(org): + if not RegistrationFormUserProfile.legal_usernames_re.search(org): raise serializers.ValidationError( _( "Organization may only contain alpha-numeric characters and " @@ -109,14 +114,14 @@ def validate_org(self, value): # pylint: disable=no-self-use except User.DoesNotExist: return org - raise serializers.ValidationError(_("Organization %s already exists." % org)) + raise serializers.ValidationError(_(f"Organization {org} already exists.")) def get_users(self, obj): # pylint: disable=no-self-use """ Return organization members. """ - def create_user_list(user_list): + def _create_user_list(user_list): return [ { "user": u.username, @@ -134,7 +139,7 @@ def create_user_list(user_list): if owners and members: members = members.exclude(username__in=[user.username for user in owners]) - members_list = create_user_list(members) - owners_list = create_user_list(owners) + members_list = _create_user_list(members) + owners_list = _create_user_list(owners) return owners_list + members_list From c1c7ba1ebca7049eadd9ccdebe903a1e43f26022 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 13:09:01 +0300 Subject: [PATCH 151/234] chart_tools.py: cleanup --- onadata/libs/utils/chart_tools.py | 81 ++++++++++++++++++------------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index 6aacfec896..2c95fc45e6 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -1,20 +1,25 @@ +# -*- coding=utf-8 -*- +""" +Chart utility functions. +""" from __future__ import unicode_literals import re -import six - -from builtins import str as text from collections import OrderedDict from django.db.utils import DataError from django.http import Http404 + +import six from rest_framework.exceptions import ParseError from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.xform import XForm -from onadata.libs.data.query import get_form_submissions_aggregated_by_select_one -from onadata.libs.data.query import get_form_submissions_grouped_by_field -from onadata.libs.data.query import get_form_submissions_grouped_by_select_one +from onadata.libs.data.query import ( + get_form_submissions_aggregated_by_select_one, + get_form_submissions_grouped_by_field, + get_form_submissions_grouped_by_select_one, +) from onadata.libs.utils import common_tags # list of fields we can chart @@ -65,23 +70,25 @@ def utc_time_string_for_javascript(date_string): match = timezone_re.match(date_string) if not match: raise ValueError( - "{} fos not match the format 2014-01-16T12:07:23.322+03".format(date_string) + f"{date_string} fos not match the format 2014-01-16T12:07:23.322+03" ) date_time = match.groups()[0] - tz = match.groups()[1] - if len(tz) == 2: - tz += "00" - elif len(tz) != 4: - raise ValueError("len of {} must either be 2 or 4") + timezone = match.groups()[1] + if len(timezone) == 2: + timezone += "00" + elif len(timezone) != 4: + raise ValueError(f"len of {timezone} must either be 2 or 4") - return "{}+{}".format(date_time, tz) + return f"{date_time}+{timezone}" def find_choice_label(choices, string): + """Returns the choice label of the given ``string``.""" for choice in choices: if choice["name"] == string: return choice["label"] + return None def get_field_choices(field, xform): @@ -147,13 +154,13 @@ def _flatten_multiple_dict_into_one(field_name, group_by_name, data): for b in list({a.get(truncated_field_name) for a in data}) ] - for a in data: - for b in final: - if a.get(truncated_field_name) == b.get(truncated_field_name): - b["items"].append( + for round_1 in data: + for round_2 in final: + if round_1.get(truncated_field_name) == round_2.get(truncated_field_name): + round_2["items"].append( { - truncated_group_by_name: a.get(truncated_group_by_name), - "count": a.get("count"), + truncated_group_by_name: round_1.get(truncated_group_by_name), + "count": round_1.get("count"), } ) @@ -215,9 +222,11 @@ def _use_labels_from_group_by_name(field_name, field, data_type, data, choices=N return data +# pylint: disable=too-many-locals,too-many-branches,too-many-arguments def build_chart_data_for_field( xform, field, language_index=0, choices=None, group_by=None, data_view=None ): + """Returns the chart data for a given field.""" # check if its the special _submission_time META if isinstance(field, str): field_label, field_xpath, field_type = FIELD_DATA_MAP.get(field) @@ -276,7 +285,7 @@ def build_chart_data_for_field( xform, field_xpath, field_name, group_by_name, data_view ) else: - raise ParseError("Cannot group by %s" % group_by_name) + raise ParseError(f"Cannot group by {group_by_name}") else: result = get_form_submissions_grouped_by_field( xform, field_xpath, field_name, data_view @@ -293,15 +302,15 @@ def build_chart_data_for_field( group_by_name, group_by, group_by_data_type, result, choices=grp_choices ) elif group_by and isinstance(group_by, list): - for g in group_by: - if isinstance(g, six.string_types): + for a_group in group_by: + if isinstance(a_group, six.string_types): continue - group_by_data_type = DATA_TYPE_MAP.get(g.type, "categorized") - grp_choices = get_field_choices(g, xform) + group_by_data_type = DATA_TYPE_MAP.get(a_group.type, "categorized") + grp_choices = get_field_choices(a_group, xform) result = _use_labels_from_group_by_name( - g.get_abbreviated_xpath(), - g, + a_group.get_abbreviated_xpath(), + a_group, group_by_data_type, result, choices=grp_choices, @@ -314,10 +323,10 @@ def build_chart_data_for_field( if data_type == "time_based": result = [r for r in result if r.get(field_name) is not None] # for each check if it matches the timezone regexp and convert for js - for r in result: - if timezone_re.match(r[field_name]): + for row in result: + if timezone_re.match(row[field_name]): try: - r[field_name] = utc_time_string_for_javascript(r[field_name]) + row[field_name] = utc_time_string_for_javascript(row[field_name]) except ValueError: pass @@ -343,6 +352,7 @@ def calculate_ranges(page, items_per_page, total_items): def build_chart_data(xform, language_index=0, page=0): + """Returns chart data for all the fields in the ``xform``.""" # only use chart-able fields fields = [e for e in xform.survey_elements if e.type in CHART_FIELDS] @@ -360,6 +370,7 @@ def build_chart_data(xform, language_index=0, page=0): def build_chart_data_from_widget(widget, language_index=0): + """Returns chart data from a widget.""" if isinstance(widget.content_object, XForm): xform = widget.content_object @@ -378,7 +389,7 @@ def build_chart_data_from_widget(widget, language_index=0): fields = [e for e in xform.survey_elements if e.name == field_name] if len(fields) == 0: - raise ParseError("Field %s does not not exist on the form" % field_name) + raise ParseError(f"Field {field_name} does not not exist on the form") field = fields[0] choices = xform.survey.get("choices") @@ -388,7 +399,7 @@ def build_chart_data_from_widget(widget, language_index=0): try: data = build_chart_data_for_field(xform, field, language_index, choices=choices) except DataError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e return data @@ -405,22 +416,25 @@ def _get_field_from_field_fn(field_str, xform, field_fn): # use specified field to get summary fields = [e for e in xform.survey_elements if field_fn(e) == field_str] if len(fields) == 0: - raise Http404("Field %s does not not exist on the form" % field_str) + raise Http404(f"Field {field_str} does not not exist on the form") field = fields[0] return field def get_field_from_field_name(field_name, xform): + """Returns the field if the ``field_name`` is in the ``xform``.""" return _get_field_from_field_fn(field_name, xform, lambda x: x.name) def get_field_from_field_xpath(field_xpath, xform): + """Returns the field if the ``field_xpath`` is in the ``xform``.""" return _get_field_from_field_fn( field_xpath, xform, lambda x: x.get_abbreviated_xpath() ) def get_field_label(field, language_index=0): + """Returns the ``field``'s label or name based on selected ``language_index``.'""" # check if label is dict i.e. multilang if isinstance(field.label, dict) and len(list(field.label)) > 0: languages = list(OrderedDict(field.label)) @@ -432,6 +446,7 @@ def get_field_label(field, language_index=0): return field_label +# pylint: disable=too-many-arguments def get_chart_data_for_field( field_name, xform, accepted_format, group_by, field_xpath=None, data_view=None ): @@ -461,7 +476,7 @@ def get_chart_data_for_field( xform, field, choices=choices, group_by=group_by, data_view=data_view ) except DataError as e: - raise ParseError(text(e)) + raise ParseError(str(e)) from e else: if accepted_format == "json" or not accepted_format: xform = xform.pk From 25cfa6ae44ac2e7390e70e706bdc218d8839da22 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 13:27:20 +0300 Subject: [PATCH 152/234] signals.py: cleanup --- onadata/libs/mixins/labels_mixin.py | 7 ++++--- onadata/libs/models/signals.py | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/onadata/libs/mixins/labels_mixin.py b/onadata/libs/mixins/labels_mixin.py index 57648b23c9..79925d0ed0 100644 --- a/onadata/libs/mixins/labels_mixin.py +++ b/onadata/libs/mixins/labels_mixin.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- """LabelMixin module""" from django import forms + from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response from taggit.forms import TagField from onadata.apps.logger.models import XForm -from onadata.libs.models.signals import XFORM_TAGS_ADD, XFORM_TAGS_DELETE +from onadata.libs.models.signals import xform_tags_add, xform_tags_delete class TagForm(forms.Form): @@ -33,7 +34,7 @@ def _labels_post(request, instance): instance.tags.add(tag) if isinstance(instance, XForm): - XFORM_TAGS_ADD.send(sender=XForm, xform=instance, tags=tags) + xform_tags_add.send(sender=XForm, xform=instance, tags=tags) return status.HTTP_201_CREATED return None @@ -51,7 +52,7 @@ def _labels_delete(label, instance): instance.tags.remove(label) if isinstance(instance, XForm): - XFORM_TAGS_DELETE.send(sender=XForm, xform=instance, tag=label) + xform_tags_delete.send(sender=XForm, xform=instance, tag=label) # Accepted, label does not exist hence nothing removed http_status = ( diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 2fb10a693b..1307e77981 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -4,12 +4,12 @@ from onadata.apps.logger.models import XForm -XFORM_TAGS_ADD = django.dispatch.Signal(providing_args=["xform", "tags"]) -XFORM_TAGS_DELETE = django.dispatch.Signal(providing_args=["xform", "tag"]) +xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) +xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) # pylint: disable=unused-argument -@django.dispatch.receiver(XFORM_TAGS_ADD, sender=XForm) +@django.dispatch.receiver(xform_tags_add, sender=XForm) def add_tags_to_xform_instances(sender, **kwargs): """Adds tags to an xform instance.""" xform = kwargs.get("xform", None) @@ -25,7 +25,7 @@ def add_tags_to_xform_instances(sender, **kwargs): # pylint: disable=unused-argument -@django.dispatch.receiver(XFORM_TAGS_DELETE, sender=XForm) +@django.dispatch.receiver(xform_tags_delete, sender=XForm) def delete_tag_from_xform_instances(sender, **kwargs): """Deletes tags associated with an xform when it is deleted.""" xform = kwargs.get("xform", None) From eed0184ffb2aa20b2f858bfcb6974f7dc1dcd580 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 13:41:21 +0300 Subject: [PATCH 153/234] change_s3_media_permissions.py: cleanup --- .../commands/change_s3_media_permissions.py | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 7ca4be7068..7fa04adc28 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -1,17 +1,24 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 -import sys +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +change_s3_media_permissions - makes all s3 files private. +""" -from django.core.management.base import BaseCommand, CommandError from django.core.files.storage import get_storage_class -from django.utils.translation import gettext as _, gettext_lazy +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy class Command(BaseCommand): + """Makes all s3 files private""" + help = gettext_lazy("Makes all s3 files private") def handle(self, *args, **kwargs): - permissions = ('private', 'public-read', 'authenticated-read') + """Makes all s3 files private""" + permissions = ("private", "public-read", "authenticated-read") if len(args) < 1: raise CommandError(_("Missing permission argument")) @@ -19,24 +26,16 @@ def handle(self, *args, **kwargs): permission = args[0] if permission not in permissions: - raise CommandError(_( - "Expected %s as permission") % ' or '.join(permissions)) - - try: - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() - except Exception: - self.stderr.write(_( - u"Missing necessary libraries. Try running: pip install " - "-r requirements-s3.pip")) - sys.exit(1) - else: - all_files = s3.bucket.list() - - for i, f in enumerate(all_files): - f.set_acl(permission) - if i % 1000 == 0: - self.stdout.write(_( - "%s file objects processed" % i)) - - self.stdout.write(_( - "A total of %s file objects processed" % i)) + raise CommandError(_(f"Expected {' or '.join(permissions)} as permission")) + + s3_storage = get_storage_class("storages.backends.s3boto.S3BotoStorage")() + all_files = s3_storage.bucket.list() + + num = 0 + for i, f in enumerate(all_files): + f.set_acl(permission) + if i % 1000 == 0: + self.stdout.write(_(f"{i} file objects processed")) + num = i + + self.stdout.write(_(f"A total of {num} file objects processed")) From 9f48aed09a58b471e9d0b4e3b1373aff9a678dc7 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 13:46:52 +0300 Subject: [PATCH 154/234] export_xforms_and_instances.py: cleanup --- .../commands/export_xforms_and_instances.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/onadata/apps/logger/management/commands/export_xforms_and_instances.py b/onadata/apps/logger/management/commands/export_xforms_and_instances.py index a6c714f160..28c7088473 100644 --- a/onadata/apps/logger/management/commands/export_xforms_and_instances.py +++ b/onadata/apps/logger/management/commands/export_xforms_and_instances.py @@ -1,5 +1,9 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +export_xformx_and_instances - exports XForms and submission instances into JSON files. +""" import os from django.core.management.base import BaseCommand @@ -12,9 +16,12 @@ class Command(BaseCommand): + """Export ODK forms and instances to JSON.""" + help = gettext_lazy("Export ODK forms and instances to JSON.") def handle(self, *args, **kwargs): + """Export ODK forms and instances to JSON.""" fixtures_dir = os.path.join(PROJECT_ROOT, "json_xform_fixtures") if not os.path.exists(fixtures_dir): os.mkdir(fixtures_dir) @@ -22,10 +29,10 @@ def handle(self, *args, **kwargs): xform_fp = os.path.join(fixtures_dir, "a-xforms.json") instance_fp = os.path.join(fixtures_dir, "b-instances.json") - xfp = open(xform_fp, 'w') - xfp.write(serialize("json", XForm.objects.all())) - xfp.close() + with open(xform_fp, "w", encoding="utf-8") as xfp: + xfp.write(serialize("json", XForm.objects.all())) + xfp.close() - ifp = open(instance_fp, 'w') - ifp.write(serialize("json", Instance.objects.all())) - ifp.close() + with open(instance_fp, "w", encoding="utf-8") as ifp: + ifp.write(serialize("json", Instance.objects.all())) + ifp.close() From e7aa70307fdbb9bf4bba4df372b8f9a25814c922 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:02:51 +0300 Subject: [PATCH 155/234] update_enketo_urls.py: cleanup --- .../apps/main/management/commands/update_enketo_urls.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 96a66d06fd..14c7c93fde 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- -"""update_enketo_urls - command to update Enketo preview URLs in the MetaData model.""" +""" +update_enketo_urls - command to update Enketo preview URLs in the MetaData model. +""" +import argparse import os +import sys from django.core.management.base import BaseCommand, CommandError from django.db.models import Q @@ -28,6 +32,9 @@ def add_arguments(self, parser): dest="generate_consistent_urls", default=True, ) + parser.add_argument( + "enketo_urls_file", argparse.FileType("r"), default=sys.stdin + ) # pylint: disable=too-many-locals def handle(self, *args, **options): From 6ea1e0494063410c9794a4cfd9e69b208beeb87e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:13:32 +0300 Subject: [PATCH 156/234] update_enketo_urls.py: cleanup --- .../management/commands/update_enketo_urls.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 14c7c93fde..42e61b3b52 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -33,7 +33,7 @@ def add_arguments(self, parser): default=True, ) parser.add_argument( - "enketo_urls_file", argparse.FileType("r"), default=sys.stdin + "enketo_urls_file", argparse.FileType("w"), default=sys.stdout ) # pylint: disable=too-many-locals @@ -44,6 +44,7 @@ def handle(self, *args, **options): server_port = options.get("server_port") protocol = options.get("protocol") generate_consistent_urls = options.get("generate_consistent_urls") + enketo_urls_file = options.get("enketo_urls_file") if not server_name or not server_port or not protocol: raise CommandError( @@ -80,24 +81,21 @@ def handle(self, *args, **options): if not os.path.exists("/tmp/enketo_url"): break - with open("/tmp/enketo_url", "a", encoding="utf-8") as f: - form_url = get_form_url( - request, - username=username, - xform_pk=xform_pk, - generate_consistent_urls=generate_consistent_urls, - ) - enketo_urls = get_enketo_urls(form_url, id_string) - if data_type == "enketo_url": - _enketo_url = enketo_urls.get("offline_url") or enketo_urls.get( - "url" - ) - MetaData.enketo_url(xform, _enketo_url) - elif data_type == "enketo_preview_url": - _enketo_preview_url = enketo_urls.get("preview_url") - MetaData.enketo_preview_url(xform, _enketo_preview_url) - - f.write(f"{id_string} : {data_value} \n") + form_url = get_form_url( + request, + username=username, + xform_pk=xform_pk, + generate_consistent_urls=generate_consistent_urls, + ) + enketo_urls = get_enketo_urls(form_url, id_string) + if data_type == "enketo_url": + _enketo_url = enketo_urls.get("offline_url") or enketo_urls.get("url") + MetaData.enketo_url(xform, _enketo_url) + elif data_type == "enketo_preview_url": + _enketo_preview_url = enketo_urls.get("preview_url") + MetaData.enketo_preview_url(xform, _enketo_preview_url) + + enketo_urls_file.write(f"{id_string} : {data_value} \n") self.stdout.write(f"{data_type}: {meta_data.data_value}") self.stdout.write("enketo urls update complete!!") From 8738fa393b38a04fe1203112c56ebe02df495cb0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:20:58 +0300 Subject: [PATCH 157/234] main/urls.py: cleanup --- onadata/apps/main/urls.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index 988889c5ab..bb51a9941b 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -1,33 +1,37 @@ +# -*- coding: utf-8 -*- +""" +URLs path. +""" import sys import django - from django.conf import settings -from django.urls import include, re_path from django.conf.urls import i18n + +# enable the admin: +from django.contrib import admin from django.contrib.staticfiles import views as staticfiles_views +from django.urls import include, re_path from django.views.generic import RedirectView from onadata.apps import sms_support +from onadata.apps.api.urls.v1_urls import ( + BriefcaseViewset, + XFormListViewSet, + XFormSubmissionViewSet, +) from onadata.apps.api.urls.v1_urls import router as api_v1_router from onadata.apps.api.urls.v2_urls import router as api_v2_router -from onadata.apps.api.urls.v1_urls import XFormListViewSet from onadata.apps.api.viewsets.xform_list_viewset import PreviewXFormListViewSet -from onadata.apps.api.urls.v1_urls import XFormSubmissionViewSet -from onadata.apps.api.urls.v1_urls import BriefcaseViewset +from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger import views as logger_views from onadata.apps.main import views as main_views from onadata.apps.main.registration_urls import urlpatterns as registration_patterns from onadata.apps.restservice import views as restservice_views from onadata.apps.sms_support import views as sms_support_views from onadata.apps.viewer import views as viewer_views -from onadata.apps.api.viewsets.xform_viewset import XFormViewSet - from onadata.libs.utils.analytics import init_analytics -# enable the admin: -from django.contrib import admin - TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" admin.autodiscover() @@ -166,6 +170,7 @@ name="delete-metadata", ), re_path( + # pylint: disable=line-too-long r"^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)", main_views.download_media_data, # noqa name="download-media-data", @@ -324,6 +329,7 @@ name="form-version-detail", ), re_path( + # pylint: disable=line-too-long r"^api/v1/forms/(?P[^/.]+)/versions/(?P[^/.]+)\.(?P[a-z0-9]+)/?$", # noqa XFormViewSet.as_view({"get": "versions"}), name="form-version-detail", @@ -385,6 +391,7 @@ name="xform-media", ), re_path( + # pylint: disable=line-too-long r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$", # noqa XFormListViewSet.as_view({"get": "media", "head": "media"}), name="xform-media", @@ -451,6 +458,7 @@ ), # SMS support re_path( + # pylint: disable=line-too-long r"^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$", # noqa sms_support.providers.import_submission_for_form, name="sms_submission_form_api", From f983aa4484b206538fbaa7aa49524ce022e7e01e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:34:31 +0300 Subject: [PATCH 158/234] viewer_tools.py: use defusedxml.minidom --- onadata/libs/utils/viewer_tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index ee3e2498cf..7db814717b 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -7,7 +7,7 @@ from json.decoder import JSONDecodeError from tempfile import NamedTemporaryFile from typing import Dict -from xml.dom import minidom +from defusedxml import minidom from django.conf import settings from django.core.files.storage import get_storage_class @@ -154,7 +154,7 @@ def get_client_ip(request): arguments: request -- HttpRequest object. """ - x_forwarded_for = request.headers.get('X-Forwarded-For') + x_forwarded_for = request.headers.get("X-Forwarded-For") if x_forwarded_for: return x_forwarded_for.split(",")[0] @@ -321,7 +321,7 @@ def get_form_url( http_host = settings.TEST_HTTP_HOST username = settings.TEST_USERNAME else: - http_host = request.headers.get('Host', "ona.io") + http_host = request.headers.get("Host", "ona.io") url = f"{protocol}://{http_host}" From a8fb5a0b118f104cfddbea7f9d21233c068e3731 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:41:49 +0300 Subject: [PATCH 159/234] xform_viewset.py: cleanup --- onadata/apps/api/viewsets/xform_viewset.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 646bc5c22f..f86949d582 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- +""" +The /forms API endpoint. +""" import json import os import random from datetime import datetime -import six -from six.moves.urllib.parse import urlparse - from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError @@ -27,7 +27,9 @@ from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache +import six from django_filters.rest_framework import DjangoFilterBackend +from six.moves.urllib.parse import urlparse try: from multidb.pinning import use_master @@ -355,7 +357,9 @@ def get_serializer_class(self): return super().get_serializer_class() + # pylint: disable=unused-argument def create(self, request, *args, **kwargs): + """Support XLSForm publishing endpoint `POST /api/v1/forms`.""" try: owner = _get_owner(request) except ValidationError as e: @@ -376,6 +380,7 @@ def create(self, request, *args, **kwargs): return Response(survey, status=status.HTTP_400_BAD_REQUEST) + # pylint: disable=unused-argument @action(methods=["POST", "GET"], detail=False) def create_async(self, request, *args, **kwargs): """Temporary Endpoint for Async form creation""" @@ -446,6 +451,7 @@ def form(self, request, **kwargs): return response # pylint: disable=no-self-use + # pylint: disable=unused-argument @action(methods=["GET"], detail=False) def login(self, request, **kwargs): """Authenticate and redirect to URL in `return` query parameter.""" @@ -467,6 +473,7 @@ def login(self, request, **kwargs): return HttpResponseForbidden("Authentication failure, cannot redirect") + # pylint: disable=unused-argument @action(methods=["GET"], detail=True) def enketo(self, request, **kwargs): """Expose enketo urls.""" @@ -955,6 +962,7 @@ def get_json_string(item): return response def list(self, request, *args, **kwargs): + """List forms API endpoint `GET /api/v1/forms`.""" stream_data = getattr(settings, "STREAM_DATA", False) try: queryset = self.filter_queryset(self.get_queryset()) From 4780e4649cddf2b78df56b1f534bc4e3b1bdaa0a Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 14:44:48 +0300 Subject: [PATCH 160/234] labels_mixin.py: cleanup --- onadata/libs/mixins/labels_mixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/mixins/labels_mixin.py b/onadata/libs/mixins/labels_mixin.py index 79925d0ed0..836c16672a 100644 --- a/onadata/libs/mixins/labels_mixin.py +++ b/onadata/libs/mixins/labels_mixin.py @@ -100,7 +100,7 @@ class LabelsMixin: "label", ], ) - def labels(self, request, format="json", **kwargs): # noqa + def labels(self, request, **kwargs): """Process request to labels endpoint. :param request: HTTP request object. From 8835c1b5f91975d9431a11a8a55328f43d89ef86 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 15:12:14 +0300 Subject: [PATCH 161/234] tools.py: cleanup --- onadata/apps/sms_support/tools.py | 82 +++++++++++++++++-------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index d0690c31aa..8e9cc5e810 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -1,24 +1,29 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support utility functions module. +""" import copy import io import mimetypes from xml.parsers.expat import ExpatError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.uploadedfile import InMemoryUploadedFile from django.http import HttpRequest +from django.urls import reverse from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm from onadata.apps.logger.models.instance import FormInactiveError -from onadata.apps.logger.xform_instance_parser import DuplicateInstance -from onadata.apps.logger.xform_instance_parser import InstanceEmptyError -from onadata.apps.logger.xform_instance_parser import InstanceInvalidUserError -from onadata.libs.utils.log import Actions -from onadata.libs.utils.log import audit_log +from onadata.apps.logger.xform_instance_parser import ( + DuplicateInstance, + InstanceEmptyError, + InstanceInvalidUserError, +) +from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import create_instance SMS_API_ERROR = "SMS_API_ERROR" @@ -27,11 +32,10 @@ SMS_SUBMISSION_REFUSED = "SMS_SUBMISSION_REFUSED" SMS_INTERNAL_ERROR = "SMS_INTERNAL_ERROR" -BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz0123456789+/=" +BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" DEFAULT_SEPARATOR = "+" DEFAULT_ALLOW_MEDIA = False NA_VALUE = "n/a" -BASE64_ALPHABET = None META_FIELDS = ( "start", "end", @@ -46,8 +50,12 @@ DEFAULT_DATETIME_FORMAT = "%Y-%m-%d-%H:%M" SENSITIVE_FIELDS = ("text", "select all that apply", "geopoint", "barcode") +# pylint: disable=invalid-name +User = get_user_model() + def is_last(index, items): + """Returns True if ``index`` is the last index in ``items``.""" return index == len(items) - 1 or ( items[-1].get("type") == "note" and index == len(items) - 2 ) @@ -63,6 +71,7 @@ def get_sms_instance_id(instance): def sms_media_to_file(file_object, name): + """Returns a file object from an SMS string.""" if isinstance(file_object, str): file_object = io.BytesIO(file_object) @@ -86,6 +95,7 @@ def getsize(f): ) +# pylint: disable=too-many-return-statements def generate_instance(username, xml_file, media_files, uuid=None): """Process an XForm submission as if done via HTTP @@ -103,7 +113,7 @@ def generate_instance(username, xml_file, media_files, uuid=None): except InstanceEmptyError: return { "code": SMS_INTERNAL_ERROR, - "text": _("Received empty submission. " "No instance was created"), + "text": _("Received empty submission. No instance was created"), } except FormInactiveError: return {"code": SMS_SUBMISSION_REFUSED, "text": _("Form is not active")} @@ -127,15 +137,15 @@ def generate_instance(username, xml_file, media_files, uuid=None): Actions.SUBMISSION_CREATED, user, instance.xform.user, - _("Created submission on form %(id_string)s.") - % {"id_string": instance.xform.id_string}, + _(f"Created submission on form {instance.xform.id_string}."), audit, HttpRequest(), ) xml_file.close() - if len(media_files): - [_file.close() for _file in media_files] + if media_files: + for _file in media_files: + _file.close() return { "code": SMS_SUBMISSION_ACCEPTED, @@ -149,29 +159,31 @@ def is_sms_related(json_survey): return True if one sms-related field is defined.""" - def treat(value, key=None): + def _treat(value, key=None): if key is None: return False if key in ("sms_field", "sms_option") and value: if not value.lower() in ("no", "false"): return True + return False - def walk(dl): + def _walk(dl): if not isinstance(dl, (dict, list)): return False iterator = [(None, e) for e in dl] if isinstance(dl, list) else dl.items() for k, v in iterator: if k == "parent": continue - if treat(v, k): + if _treat(v, k): return True - if walk(v): + if _walk(v): return True return False - return walk(json_survey) + return _walk(json_survey) +# pylint: disable=too-many-locals,too-many-branches def check_form_sms_compatibility(form, json_survey=None): """Tests all SMS related rules on the XForm representation @@ -182,16 +194,13 @@ def check_form_sms_compatibility(form, json_survey=None): json_survey = form.get("form_o", {}) def prep_return(msg, comp=None): - - from django.urls import reverse - error = "alert-info" warning = "alert-info" success = "alert-success" + syntax_url = reverse("syntax") outro = ( - '
    Please check the ' - "SMS Syntax Page." % {"syntax_url": reverse("syntax")} + f'
    Please check the ' + "SMS Syntax Page." ) # no compatibility at all @@ -207,7 +216,7 @@ def prep_return(msg, comp=None): elif comp == 1: alert = warning msg = "%(prefix)s
      %(msg)s
    " % { - "prefix": "Your form can be used with SMS, " "knowing that:", + "prefix": "Your form can be used with SMS, knowing that:", "msg": msg, } # SMS compatible @@ -252,7 +261,7 @@ def prep_return(msg, comp=None): _( "All your groups must have an 'sms_field' " "(use 'meta' prefixed ones for non-fillable " - "groups). %s" % bad_groups[-1] + f"groups). {bad_groups[-1]}" ) ) # all select_one or select_multiple fields muts have sms_option for each. @@ -265,7 +274,7 @@ def prep_return(msg, comp=None): if xlsf_type in ("select one", "select all that apply"): nb_choices = len(xlsf_choices) options = list( - set([c.get("sms_option", "") or None for c in xlsf_choices]) + set(c.get("sms_option", "") or None for c in xlsf_choices) ) try: options.remove(None) @@ -276,10 +285,9 @@ def prep_return(msg, comp=None): return prep_return( _( "Not all options in the choices list for " - "%s have an " + f"{xlsf_name} have an " "sms_option value." ) - % xlsf_name ) # has sensitive (space containing) fields in non-last position @@ -296,7 +304,7 @@ def prep_return(msg, comp=None): _( "Questions for which values can contain " "spaces are only allowed on last " - "position of group (%s)" % field.get("name") + f"position of group ({field.get('name')})" ) ) # separator is not set or is within BASE64 alphabet and sms_allow_media @@ -308,9 +316,9 @@ def prep_return(msg, comp=None): return prep_return( _( "When allowing medias ('sms_allow_media'), your " - "separator (%s) must be outside Base64 alphabet " + f"separator ({separator}) must be outside Base64 alphabet " "(letters, digits and +/=). " - "You case use '#' instead." % separator + "You case use '#' instead." ) ) @@ -329,7 +337,7 @@ def prep_return(msg, comp=None): warnings.append( "
  • You have 'date' fields without " "explicitly setting a date format. " - "Default (%s) will be used.
    • " % DEFAULT_DATE_FORMAT + f"Default ({DEFAULT_DATE_FORMAT}) will be used." ) break # has datetime field with no datetime format @@ -341,7 +349,7 @@ def prep_return(msg, comp=None): warnings.append( "
  • You have 'datetime' fields without " "explicitly setting a datetime format. " - "Default (%s) will be used.
    • " % DEFAULT_DATETIME_FORMAT + f"Default ({DEFAULT_DATETIME_FORMAT}) will be used." ) break @@ -350,16 +358,16 @@ def prep_return(msg, comp=None): warnings.append( "
  • 'sms_date_format' contains space which will " "require 'date' questions to be positioned at " - "the end of groups (%s).
    • " % date_format + f"the end of groups ({date_format})." ) if "datetime" in sensitive_fields: warnings.append( "
  • 'sms_datetime_format' contains space which will " "require 'datetime' questions to be positioned at " - "the end of groups (%s).
    • " % datetime_format + f"the end of groups ({datetime_format})." ) - if len(warnings): + if warnings: return prep_return("".join(warnings), comp=1) # Good to go From 7dba4eae8fd6a0117ac68b50c66a8667122f029a Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 15:27:34 +0300 Subject: [PATCH 162/234] move_media_to_s3.py: cleanup --- .../management/commands/move_media_to_s3.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index 08fed5b6da..17fa12f08a 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +move_media_to_s3 - Moves all XLSForm file from local storage to S3 storage. +""" import sys from django.core.files.storage import get_storage_class @@ -11,26 +15,20 @@ class Command(BaseCommand): + """Moves all XLSForm file from local storage to S3 storage.""" + help = gettext_lazy( "Moves all attachments and xls files " "to s3 from the local file system storage." ) def handle(self, *args, **kwargs): - try: - fs = get_storage_class("django.core.files.storage.FileSystemStorage")() - s3 = get_storage_class("storages.backends.s3boto.S3BotoStorage")() - except Exception: - self.stderr.write( - _( - "Missing necessary libraries. Try running: pip install -r" - "requirements/s3.pip" - ) - ) - sys.exit(1) + """Moves all XLSForm file from local storage to S3 storage.""" + local_fs = get_storage_class("django.core.files.storage.FileSystemStorage")() + s3_fs = get_storage_class("storages.backends.s3boto.S3BotoStorage")() default_storage = get_storage_class()() - if default_storage.__class__ != s3.__class__: + if default_storage.__class__ != s3_fs.__class__: self.stderr.write( _( "You must first set your default storage to s3 in your " @@ -49,19 +47,20 @@ def handle(self, *args, **kwargs): for i in cls.objects.all(): f = getattr(i, file_field) old_filename = f.name - if f.name and fs.exists(f.name) and not s3.exists(upload_to(i, f.name)): - f.save(fs.path(f.name), fs.open(fs.path(f.name))) + if ( + f.name + and local_fs.exists(f.name) + and not s3_fs.exists(upload_to(i, f.name)) + ): + f.save(local_fs.path(f.name), local_fs.open(local_fs.path(f.name))) self.stdout.write( _("\t+ '%(fname)s'\n\t---> '%(url)s'") - % {"fname": fs.path(old_filename), "url": f.url} + % {"fname": local_fs.path(old_filename), "url": f.url} ) else: + exists_locally = local_fs.exists(f.name) + exists_s3 = not s3_fs.exists(upload_to(i, f.name)) self.stderr.write( - "\t- (f.name=%s, fs.exists(f.name)=%s, not s3.exist" - "s(upload_to(i, f.name))=%s)" - % ( - f.name, - fs.exists(f.name), - not s3.exists(upload_to(i, f.name)), - ) + f"\t- (f.name={f.name}, fs.exists(f.name)={exists_locally}," + f" not s3.exist s3upload_to(i, f.name))={exists_s3})" ) From 2fcd1b4edccc22e973caaa2ba6e844cc5e328f1e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 15:50:48 +0300 Subject: [PATCH 163/234] filters.py: cleanup --- .../management/commands/move_media_to_s3.py | 1 + onadata/libs/filters.py | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index 17fa12f08a..a123cc1e5d 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -22,6 +22,7 @@ class Command(BaseCommand): "to s3 from the local file system storage." ) + # pylint: disable=unused-argument def handle(self, *args, **kwargs): """Moves all XLSForm file from local storage to S3 storage.""" local_fs = get_storage_class("django.core.files.storage.FileSystemStorage")() diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 37b12c8450..9b1041a95a 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +Django rest_framework ViewSet filters. +""" from uuid import UUID import six @@ -121,8 +124,10 @@ def filter_queryset(self, request, queryset, view): class FormIDFilter(django_filter_filters.FilterSet): """formID filter using the XForm.id_string.""" + # pylint: disable=invalid-name formID = django_filter_filters.CharFilter(field_name="id_string") # noqa + # pylint: disable=missing-class-docstring class Meta: model = XForm fields = ["formID"] @@ -152,9 +157,11 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class XFormOwnerFilter(filters.BaseFilterBackend): + """XForm `owner` filter""" owner_prefix = "user" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): owner = request.query_params.get("owner") @@ -168,6 +175,9 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class DataFilter(ObjectPermissionsFilter): + """Shared data filter.""" + + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): if request.user.is_anonymous: return queryset.filter(Q(shared_data=True)) @@ -191,6 +201,7 @@ class InstanceFilter(django_filter_filters.FilterSet): ) media_all_received = django_filter_filters.BooleanFilter() + # pylint: disable=missing-class-docstring class Meta: model = Instance date_field_lookups = [ @@ -232,9 +243,13 @@ class Meta: # pylint: disable=too-few-public-methods class ProjectOwnerFilter(filters.BaseFilterBackend): + """Project `owner` filter.""" + owner_prefix = "organization" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Project `owner` filter.""" owner = request.query_params.get("owner") if owner: @@ -249,6 +264,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class AnonUserProjectFilter(ObjectPermissionsFilter): + """Anonymous user project filter.""" + owner_prefix = "organization" def filter_queryset(self, request, queryset, view): @@ -281,6 +298,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class TagFilter(filters.BaseFilterBackend): + """Tag filter using the `tags` query parameter.""" + def filter_queryset(self, request, queryset, view): # filter by tags if available. tags = request.query_params.get("tags", None) @@ -294,6 +313,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class XFormPermissionFilterMixin: + """XForm permission filter.""" + def _xform_filter(self, request, view, keyword): """Use XForm permissions""" @@ -323,6 +344,8 @@ def _xform_filter_queryset(self, request, queryset, view, keyword): # pylint: disable=too-few-public-methods class ProjectPermissionFilterMixin: + """Project permission filter.""" + def _project_filter(self, request, view, keyword): project_id = request.query_params.get("project") @@ -350,6 +373,8 @@ def _project_filter_queryset(self, request, queryset, view, keyword): # pylint: disable=too-few-public-methods class InstancePermissionFilterMixin: + """Instance permission filter.""" + # pylint: disable=too-many-locals def _instance_filter(self, request, view, keyword): instance_kwarg = {} @@ -401,6 +426,8 @@ def _instance_filter_queryset(self, request, queryset, view, keyword): # pylint: disable=too-few-public-methods class RestServiceFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Rest service filter.""" + def filter_queryset(self, request, queryset, view): return self._xform_filter_queryset(request, queryset, view, "xform_id") @@ -412,6 +439,8 @@ class MetaDataFilter( XFormPermissionFilterMixin, ObjectPermissionsFilter, ): + """Meta data filter.""" + def filter_queryset(self, request, queryset, view): keyword = "object_id" @@ -453,6 +482,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class AttachmentFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Attachment filter.""" + def filter_queryset(self, request, queryset, view): queryset = self._xform_filter_queryset( @@ -481,6 +512,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class AttachmentTypeFilter(filters.BaseFilterBackend): + """Attachment type filter using `type` query parameter.""" + def filter_queryset(self, request, queryset, view): attachment_type = request.query_params.get("type") @@ -494,6 +527,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class TeamOrgFilter(filters.BaseFilterBackend): + """Team organization filter using `org` query parameter""" + def filter_queryset(self, request, queryset, view): org = request.data.get("org") or request.query_params.get("org") @@ -612,7 +647,7 @@ def filter_queryset(self, request, queryset, view): request, queryset, view, "xform_id" ).exclude(*has_submitted_by_key) - old_perm_format = self.perm_format + old_perm_format = getattr(self, "perm_format") # only if request.user has access to all data # noqa pylint: disable=attribute-defined-outside-init From 142333302a698aeac91c96fb0faeee8cb4495898 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 16:00:00 +0300 Subject: [PATCH 164/234] populate_osmdata_model.py: cleanup --- .../commands/populate_osmdata_model.py | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/onadata/apps/logger/management/commands/populate_osmdata_model.py b/onadata/apps/logger/management/commands/populate_osmdata_model.py index 9125e6b3a2..867cbdd030 100644 --- a/onadata/apps/logger/management/commands/populate_osmdata_model.py +++ b/onadata/apps/logger/management/commands/populate_osmdata_model.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +populate_osmdata_model command - process OSM XML and save data in OsmData model/table. +""" from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand @@ -8,10 +12,14 @@ class Command(BaseCommand): - args = '' + """Populate OsmData model with osm info.""" + + args = "" help = gettext_lazy("Populate OsmData model with osm info.") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): + """Populate OsmData model with osm info.""" xforms = XForm.objects.filter(instances_with_osm=True) # username @@ -20,20 +28,22 @@ def handle(self, *args, **kwargs): for xform in queryset_iterator(xforms): attachments = Attachment.objects.filter( - extension=Attachment.OSM, - instance__xform=xform - ).distinct('instance') + extension=Attachment.OSM, instance__xform=xform + ).distinct("instance") count = attachments.count() - c = 0 - for a in queryset_iterator(attachments): - pk = a.instance.parsed_instance.pk - save_osm_data(pk) - c += 1 - if c % 1000 == 0: - self.stdout.write("%s:%s: Processed %s of %s." % - (xform.user.username, - xform.id_string, c, count)) - - self.stdout.write("%s:%s: processed %s of %s submissions." % - (xform.user.username, xform.id_string, c, count)) + counter = 0 + for attachment in queryset_iterator(attachments): + instance_pk = attachment.instance.parsed_instance.pk + save_osm_data(instance_pk) + counter += 1 + if counter % 1000 == 0: + self.stdout.write( + f"{xform.user.username}:{xform.id_string}: " + f"Processed {counter} of {count}." + ) + + self.stdout.write( + f"{xform.user.username}:{xform.id_string}: " + f"processed {counter} of {count} submissions." + ) From 2504e596b63c256eb82e8b015de08035dadb23c2 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 16:04:52 +0300 Subject: [PATCH 165/234] osmdata.py: cleanup --- onadata/apps/logger/models/osmdata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/onadata/apps/logger/models/osmdata.py b/onadata/apps/logger/models/osmdata.py index 91013ec62d..b269692bec 100644 --- a/onadata/apps/logger/models/osmdata.py +++ b/onadata/apps/logger/models/osmdata.py @@ -35,7 +35,10 @@ def get_tag_keys(cls, xform, field_path, include_prefix=False): Returns sorted tag keys. """ query = OsmData.objects.raw( - """SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata" INNER JOIN "logger_instance" ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" ) WHERE "logger_instance"."xform_id" = %s AND field_name = %s""", # noqa + 'SELECT DISTINCT JSONB_OBJECT_KEYS(tags) as id FROM "logger_osmdata"' + ' INNER JOIN "logger_instance"' + ' ON ( "logger_osmdata"."instance_id" = "logger_instance"."id" )' + ' WHERE "logger_instance"."xform_id" = %s AND field_name = %s', [xform.pk, field_path], ) prefix = field_path + ":" if include_prefix else "" From ad9b1ea6e5efaab2a7fea36ff4fb27eebabd23dc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 16:10:47 +0300 Subject: [PATCH 166/234] tag_list_serializer.py: cleanup --- .../libs/serializers/tag_list_serializer.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/onadata/libs/serializers/tag_list_serializer.py b/onadata/libs/serializers/tag_list_serializer.py index 205c013bb8..8a404f0d73 100644 --- a/onadata/libs/serializers/tag_list_serializer.py +++ b/onadata/libs/serializers/tag_list_serializer.py @@ -1,21 +1,26 @@ +# -*- coding: utf-8 -*- +"""Tags list serializer module.""" from django.utils.translation import gettext as _ from rest_framework import serializers class TagListSerializer(serializers.Field): + """Tags serializer - represents tags as a list of strings.""" def to_internal_value(self, data): - if not isinstance(data, list): - raise serializers.ValidationError(_(u"expected a list of data")) + """Validates the data is a list.""" + if isinstance(data, list): + return data - return data + raise serializers.ValidationError(_("expected a list of data")) - def to_representation(self, obj): - if obj is None: - return super(TagListSerializer, self).to_representation(obj) + def to_representation(self, value): + """Returns all tags linked to the object ``value`` as a list.""" + if value is None: + return super().to_representation(value) - if not isinstance(obj, list): - return [tag.name for tag in obj.all()] + if not isinstance(value, list): + return [tag.name for tag in value.all()] - return obj + return value From 1a936f540470bf25e221d2e15831b019dd4e6e7d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 16:20:03 +0300 Subject: [PATCH 167/234] share_team_project_serializer.py: cleanup --- .../share_team_project_serializer.py | 32 ++++++++++++++----- .../libs/serializers/tag_list_serializer.py | 4 ++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/onadata/libs/serializers/share_team_project_serializer.py b/onadata/libs/serializers/share_team_project_serializer.py index 7a8d2351ab..780be95304 100644 --- a/onadata/libs/serializers/share_team_project_serializer.py +++ b/onadata/libs/serializers/share_team_project_serializer.py @@ -1,52 +1,68 @@ +# -*- coding: utf-8 -*- +""" +Share projects to team functions. +""" from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.share_team_project import ShareTeamProject from onadata.libs.permissions import ROLES -from onadata.libs.serializers.fields.team_field import TeamField from onadata.libs.serializers.fields.project_field import ProjectField +from onadata.libs.serializers.fields.team_field import TeamField class ShareTeamProjectSerializer(serializers.Serializer): + """Shares a project with a team.""" + team = TeamField() project = ProjectField() role = serializers.CharField(max_length=50) + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.team = validated_data.get('team', instance.team) - instance.project = validated_data.get('project', instance.project) - instance.role = validated_data.get('role', instance.role) + """Update project sharing properties.""" + instance.team = validated_data.get("team", instance.team) + instance.project = validated_data.get("project", instance.project) + instance.role = validated_data.get("role", instance.role) instance.save() return instance + # pylint: disable=no-self-use def create(self, validated_data): + """Shares a project to a team.""" instance = ShareTeamProject(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value class RemoveTeamFromProjectSerializer(ShareTeamProjectSerializer): + """Remove a team from a project.""" + remove = serializers.BooleanField() + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.remove = validated_data.get('remove', instance.remove) + """Remove a team from a project""" + instance.remove = validated_data.get("remove", instance.remove) instance.save() return instance + # pylint: disable=no-self-use def create(self, validated_data): + """Remove a team from a project""" instance = ShareTeamProject(**validated_data) instance.save() diff --git a/onadata/libs/serializers/tag_list_serializer.py b/onadata/libs/serializers/tag_list_serializer.py index 8a404f0d73..ed6d184b0e 100644 --- a/onadata/libs/serializers/tag_list_serializer.py +++ b/onadata/libs/serializers/tag_list_serializer.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- -"""Tags list serializer module.""" +""" +Tags list serializer module. +""" from django.utils.translation import gettext as _ from rest_framework import serializers From 6be367f07ca36d64e04e4272777f5211b973e61b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 17:02:34 +0300 Subject: [PATCH 168/234] fix: check if overwrite is a str --- onadata/apps/api/viewsets/xform_viewset.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index f86949d582..a55135c8e2 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -703,7 +703,11 @@ def data_import(self, request, *args, **kwargs): if xls_file and xls_file.name.split(".")[-1] in XLS_EXTENSIONS: csv_file = submission_xls_to_csv(xls_file) overwrite = request.query_params.get("overwrite") - overwrite = overwrite is not None and overwrite.lower() == "true" + overwrite = ( + overwrite.lower() == "true" + if isinstance(overwrite, str) + else overwrite + ) size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD try: csv_size = csv_file.size @@ -773,7 +777,11 @@ def csv_import(self, request, *args, **kwargs): resp.update({"error": "csv_file not a csv file"}) else: overwrite = request.query_params.get("overwrite") - overwrite = overwrite.lower() == "true" + overwrite = ( + overwrite.lower() == "true" + if isinstance(overwrite, str) + else overwrite + ) size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: resp.update( From e2c71c5890c291b5401458dc83bcf75c68442c89 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 18:57:04 +0300 Subject: [PATCH 169/234] batch: cleanup --- .pylintrc | 2 +- .../logger/management/commands/publish_xls.py | 48 ++++++++++++------- .../management/commands/restore_backup.py | 14 +++--- onadata/libs/filters.py | 23 ++++++++- onadata/libs/models/signals.py | 1 + .../libs/serializers/project_serializer.py | 3 ++ onadata/libs/utils/chart_tools.py | 8 ++-- onadata/libs/utils/viewer_tools.py | 18 ------- 8 files changed, 71 insertions(+), 46 deletions(-) diff --git a/.pylintrc b/.pylintrc index a40e5f2066..ab8c6a1b0a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/onadata/apps/logger/management/commands/publish_xls.py b/onadata/apps/logger/management/commands/publish_xls.py index d41479eaac..bd5f53851d 100644 --- a/onadata/apps/logger/management/commands/publish_xls.py +++ b/onadata/apps/logger/management/commands/publish_xls.py @@ -1,19 +1,26 @@ +# -*- coding: utf-8 -*- +""" +publish_xls - Publish an XLSForm command. +""" import os -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import InMemoryUploadedFile from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy + from pyxform.builder import create_survey_from_xls from onadata.apps.logger.models.project import Project from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.logger_tools import publish_xls_form from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.viewer_tools import django_file class Command(BaseCommand): + """Publish an XLSForm file.""" + args = "xls_file username project" help = gettext_lazy( "Publish an XLS file with the option of replacing an" "existing one" @@ -33,26 +40,28 @@ def add_arguments(self, parser): help=gettext_lazy("Replace existing form if any"), ) - def handle(self, *args, **options): + def handle(self, *args, **options): # noqa C901 + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: xls_filepath = options["xls_filepath"] - except KeyError: - raise CommandError(_("You must provide the path to the xls file.")) + except KeyError as e: + raise CommandError(_("You must provide the path to the xls file.")) from e # make sure path exists if not os.path.exists(xls_filepath): - raise CommandError(_("The xls file '%s' does not exist.") % xls_filepath) + raise CommandError(_(f"The xls file '{xls_filepath}' does not exist.")) try: username = options["username"] - except KeyError: + except KeyError as e: raise CommandError( _("You must provide the username to publish the form to.") - ) + ) from e # make sure user exists try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + except User.DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e # wasteful but we need to get the id_string beforehand survey = create_survey_from_xls(xls_filepath) @@ -71,10 +80,9 @@ def handle(self, *args, **options): else: raise CommandError( _( - "The form with id_string '%s' already exists, use the -r" - " option to replace it." + f"The form with id_string '{survey.id_string}' already exists," + " use the -r option to replace it." ) - % survey.id_string ) else: self.stdout.write(_("Form does NOT exist, publishing ..\n")) @@ -86,6 +94,14 @@ def handle(self, *args, **options): project = get_user_default_project(user) # publish - xls_file = django_file(xls_filepath, "xls_file", "application/vnd.ms-excel") - publish_xls_form(xls_file, user, project, id_string) - self.stdout.write(_("Done..\n")) + with open(xls_filepath, "rb") as file_object: + with InMemoryUploadedFile( + file=file_object, + field_name="xls_file", + name=file_object.name, + content_type="application/vnd.ms-excel", + size=os.path.getsize(xls_filepath), + charset=None, + ) as xls_file: + publish_xls_form(xls_file, user, project, id_string) + self.stdout.write(_("Done..\n")) diff --git a/onadata/apps/logger/management/commands/restore_backup.py b/onadata/apps/logger/management/commands/restore_backup.py index 10d91b2918..b43fdb82e3 100644 --- a/onadata/apps/logger/management/commands/restore_backup.py +++ b/onadata/apps/logger/management/commands/restore_backup.py @@ -22,23 +22,23 @@ class Command(BaseCommand): def handle(self, *args, **options): # pylint: disable=invalid-name - User = get_user_model() + User = get_user_model() # noqa N806 try: username = args[0] - except IndexError: + except IndexError as e: raise CommandError( _("You must provide the username to publish the form to.") - ) + ) from e # make sure user exists try: User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_(f"The user '{username}' does not exist.")) + except User.DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e try: input_file = args[1] - except IndexError: - raise CommandError(_("You must provide the path to restore from.")) + except IndexError as e: + raise CommandError(_("You must provide the path to restore from.")) from e else: input_file = os.path.realpath(input_file) diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 9b1041a95a..78ad7d7c1f 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -163,6 +163,7 @@ class XFormOwnerFilter(filters.BaseFilterBackend): # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Filter by `owner` query parameter.""" owner = request.query_params.get("owner") if owner: @@ -179,6 +180,7 @@ class DataFilter(ObjectPermissionsFilter): # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Filter by ``XForm.shared_data = True`` for anonymous users.""" if request.user.is_anonymous: return queryset.filter(Q(shared_data=True)) return queryset @@ -300,7 +302,9 @@ def filter_queryset(self, request, queryset, view): class TagFilter(filters.BaseFilterBackend): """Tag filter using the `tags` query parameter.""" + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): + """Tag filter using the `tags` query parameter.""" # filter by tags if available. tags = request.query_params.get("tags", None) @@ -543,7 +547,11 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class UserNoOrganizationsFilter(filters.BaseFilterBackend): + """Filter by ``orgs`` query parameter.""" + def filter_queryset(self, request, queryset, view): + """Returns all users that are not organizations when `orgs=false` + query parameter""" if str(request.query_params.get("orgs")).lower() == "false": organization_user_ids = OrganizationProfile.objects.values_list( "user__id", flat=True @@ -555,6 +563,8 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class OrganizationsSharedWithUserFilter(filters.BaseFilterBackend): + """Filters by ``shared_with`` query parameter.""" + def filter_queryset(self, request, queryset, view): """ This returns a queryset containing only organizations to which @@ -588,7 +598,10 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class WidgetFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): + """Filter to return forms shared with user.""" + def filter_queryset(self, request, queryset, view): + """Filter to return forms shared with user when ``view.action == "list"``.""" if view.action == "list": # Return widgets from xform user has perms to @@ -599,7 +612,11 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class UserProfileFilter(filters.BaseFilterBackend): + """Filter by the ``users`` query parameter.""" + def filter_queryset(self, request, queryset, view): + """Filter by the ``users`` query parameter - returns a queryset of only the users + in the users parameter when `view.action == "list"`""" if view.action == "list": users = request.GET.get("users") if users: @@ -615,7 +632,10 @@ def filter_queryset(self, request, queryset, view): # pylint: disable=too-few-public-methods class NoteFilter(filters.BaseFilterBackend): + """Notes filter by the query parameter ``instance``.""" + def filter_queryset(self, request, queryset, view): + """Notes filter by the query parameter ``instance``.""" instance_id = request.query_params.get("instance") if instance_id: @@ -634,10 +654,11 @@ def filter_queryset(self, request, queryset, view): class ExportFilter(XFormPermissionFilterMixin, ObjectPermissionsFilter): """ ExportFilter class uses permissions on the related xform to filter Export - queryesets. Also filters submitted_by a specific user. + querysets. Also filters submitted_by a specific user. """ def filter_queryset(self, request, queryset, view): + """Filter by xform permissions and submitted by user.""" has_submitted_by_key = ( Q(options__has_key="query") & Q(options__query__has_key="_submitted_by"), ) diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 1307e77981..5c395078a2 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -4,6 +4,7 @@ from onadata.apps.logger.models import XForm +# pylint: disable=unexpected-keyword-arg xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index 5f93a91e07..24c611ac3e 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -232,6 +232,7 @@ class ProjectXFormSerializer(serializers.HyperlinkedModelSerializer): name = serializers.ReadOnlyField(source="title") published_by_formbuilder = serializers.SerializerMethodField() + # pylint: disable=too-few-public-methods,missing-class-docstring class Meta: model = XForm fields = ( @@ -472,6 +473,7 @@ def validate_metadata(self, value): # pylint: disable=no-self-use return value def update(self, instance, validated_data): + """Update project properties.""" metadata = JsonField.to_json(validated_data.get("metadata")) if metadata is None: metadata = {} @@ -527,6 +529,7 @@ def update(self, instance, validated_data): }, ) def create(self, validated_data): + """Creates a project.""" metadata = validated_data.get("metadata", {}) if metadata is None: metadata = {} diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index 2c95fc45e6..046e0bc53c 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -190,7 +190,9 @@ def _use_labels_from_field_name(field_name, field, data_type, data, choices=None return data -def _use_labels_from_group_by_name(field_name, field, data_type, data, choices=None): +def _use_labels_from_group_by_name( # noqa C901 + field_name, field, data_type, data, choices=None +): # truncate field name to 63 characters to fix #354 truncated_name = field_name[0:POSTGRES_ALIAS_LENGTH] @@ -223,7 +225,7 @@ def _use_labels_from_group_by_name(field_name, field, data_type, data, choices=N # pylint: disable=too-many-locals,too-many-branches,too-many-arguments -def build_chart_data_for_field( +def build_chart_data_for_field( # noqa xform, field, language_index=0, choices=None, group_by=None, data_view=None ): """Returns the chart data for a given field.""" @@ -447,7 +449,7 @@ def get_field_label(field, language_index=0): # pylint: disable=too-many-arguments -def get_chart_data_for_field( +def get_chart_data_for_field( # noqa C901 field_name, xform, accepted_format, group_by, field_xpath=None, data_view=None ): """ diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 7db814717b..d86c7a34f4 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -11,7 +11,6 @@ from django.conf import settings from django.core.files.storage import get_storage_class -from django.core.files.uploadedfile import InMemoryUploadedFile from django.utils.translation import gettext as _ import requests @@ -122,23 +121,6 @@ def _all_attributes(node): yield pair -def django_file(path, field_name, content_type): - """Return an InMemoryUploadedFile object for file uploads.""" - # adapted from here: http://groups.google.com/group/django-users/browse_th\ - # read/thread/834f988876ff3c45/ - - # pylint: disable=consider-using-with - file_object = open(path, "rb") - return InMemoryUploadedFile( - file=file_object, - field_name=field_name, - name=file_object.name, - content_type=content_type, - size=os.path.getsize(path), - charset=None, - ) - - def export_def_from_filename(filename): """Return file extension and mimetype from filename.""" __, ext = os.path.splitext(filename) From ac0f60cc43b2f80596cf8a8415c0a6661daf344b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 21:52:01 +0300 Subject: [PATCH 170/234] user_auth.py: cleanup --- onadata/libs/utils/user_auth.py | 191 ++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 84 deletions(-) diff --git a/onadata/libs/utils/user_auth.py b/onadata/libs/utils/user_auth.py index 77782b9a4d..425ac4fe1d 100644 --- a/onadata/libs/utils/user_auth.py +++ b/onadata/libs/utils/user_auth.py @@ -1,45 +1,54 @@ +# -*- coding=utf-8 -*- +""" +User authentication utility functions. +""" import base64 import re from functools import wraps -from django.contrib.auth import authenticate -from django.contrib.auth.models import User +from django.contrib.auth import authenticate, get_user_model from django.contrib.sites.models import Site -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponse from django.shortcuts import get_object_or_404 -from rest_framework.authtoken.models import Token + from guardian.shortcuts import assign_perm, get_perms_for_model +from rest_framework.authtoken.models import Token from onadata.apps.api.models import OrganizationProfile, Team, TempToken from onadata.apps.logger.models import MergedXForm, Note, Project, XForm from onadata.apps.main.models import UserProfile from onadata.libs.utils.viewer_tools import get_form +# pylint: disable=invalid-name +User = get_user_model() + class HttpResponseNotAuthorized(HttpResponse): + """HttpResponse that sets basic authentication prompt.""" + status_code = 401 def __init__(self): HttpResponse.__init__(self) - self['WWW-Authenticate'] =\ - 'Basic realm="%s"' % Site.objects.get_current().name + self["WWW-Authenticate"] = f'Basic realm="{Site.objects.get_current().name}"' def check_and_set_user(request, username): + """Returns a User object or a string to redirect.""" if username != request.user.username: - return HttpResponseRedirect("/%s" % username) + return f"/{username}" content_user = None try: content_user = User.objects.get(username=username) except User.DoesNotExist: - return HttpResponseRedirect("/") + return "/" return content_user def set_profile_data(data, content_user): + """Set user profile data.""" # create empty profile if none exists - profile, created = UserProfile.objects\ - .get_or_create(user=content_user) + profile, _created = UserProfile.objects.get_or_create(user=content_user) location = "" if profile.city: location = profile.city @@ -52,76 +61,81 @@ def set_profile_data(data, content_user): user_instances = profile.num_of_submissions home_page = profile.home_page if home_page and re.match("http", home_page) is None: - home_page = "http://%s" % home_page + home_page = f"http://{home_page}" - data.update({ - 'location': location, - 'user_instances': user_instances, - 'home_page': home_page, - 'num_forms': num_forms, - 'forms': forms, - 'profile': profile, - 'content_user': content_user - }) + data.update( + { + "location": location, + "user_instances": user_instances, + "home_page": home_page, + "num_forms": num_forms, + "forms": forms, + "profile": profile, + "content_user": content_user, + } + ) def has_permission(xform, owner, request, shared=False): + """Checks if the ``request.user`` has the necessary permissions to an ``xform``.""" user = request.user - return shared or xform.shared_data or\ - (hasattr(request, 'session') and - request.session.get('public_link') == xform.uuid) or\ - owner == user or\ - user.has_perm('logger.view_xform', xform) or\ - user.has_perm('logger.change_xform', xform) + return ( + shared + or xform.shared_data + or ( + hasattr(request, "session") + and request.session.get("public_link") == xform.uuid + ) + or owner == user + or user.has_perm("logger.view_xform", xform) + or user.has_perm("logger.change_xform", xform) + ) def has_edit_permission(xform, owner, request, shared=False): + """Checks if the ``request.user`` has edit permissions to the ``xform``.""" user = request.user - return (shared and xform.shared_data) or owner == user or\ - user.has_perm('logger.change_xform', xform) + return ( + (shared and xform.shared_data) + or owner == user + or user.has_perm("logger.change_xform", xform) + ) def check_and_set_user_and_form(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Checks and returns an `xform` and `owner` if ``request.user`` has permission.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) owner = User.objects.get(username=username) - return [xform, owner] if has_permission(xform, owner, request)\ - else [False, False] + return [xform, owner] if has_permission(xform, owner, request) else [False, False] def check_and_set_form_by_id_string(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Checks xform by ``id_string`` and returns an `xform` if ``request.user`` + has permission.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) - return xform if has_permission(xform, xform.user, request)\ - else False + return xform if has_permission(xform, xform.user, request) else False def check_and_set_form_by_id(pk, request): + """Checks xform by ``pk`` and returns an `xform` if ``request.user`` + has permission.""" xform = get_object_or_404(XForm, pk=pk) - return xform if has_permission(xform, xform.user, request)\ - else False + return xform if has_permission(xform, xform.user, request) else False def get_xform_and_perms(username, id_string, request): - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } + """Returns the `xform` with the matching ``id_string``, and the permissions the + ``request.user`` has.""" + xform_kwargs = {"id_string__iexact": id_string, "user__username__iexact": username} xform = get_form(xform_kwargs) is_owner = xform.user == request.user - can_edit = is_owner or\ - request.user.has_perm('logger.change_xform', xform) - can_view = can_edit or\ - request.user.has_perm('logger.view_xform', xform) + can_edit = is_owner or request.user.has_perm("logger.change_xform", xform) + can_view = can_edit or request.user.has_perm("logger.view_xform", xform) return [xform, is_owner, can_edit, can_view] @@ -132,9 +146,11 @@ def get_xform_users_with_perms(xform): expensive as the one in use with get_users_with_perms """ user_perms = {} - for p in xform.xformuserobjectpermission_set.all().select_related( - 'user', 'permission').only('user', 'permission__codename', - 'content_object_id'): + for p in ( + xform.xformuserobjectpermission_set.all() + .select_related("user", "permission") + .only("user", "permission__codename", "content_object_id") + ): if p.user.username not in user_perms: user_perms[p.user] = [] user_perms[p.user].append(p.permission.codename) @@ -143,72 +159,81 @@ def get_xform_users_with_perms(xform): def helper_auth_helper(request): + """Authenticates a user via basic authentication.""" if request.user and request.user.is_authenticated: return None # source, http://djangosnippets.org/snippets/243/ - if 'HTTP_AUTHORIZATION' in request.META: - auth = request.headers['Authorization'].split() + if "HTTP_AUTHORIZATION" in request.META: + auth = request.headers["Authorization"].split() if len(auth) == 2 and auth[0].lower() == "basic": - uname, passwd = base64.b64decode(auth[1].encode( - 'utf-8')).decode('utf-8').split(':') + uname, passwd = ( + base64.b64decode(auth[1].encode("utf-8")).decode("utf-8").split(":") + ) user = authenticate(username=uname, password=passwd) if user: request.user = user return None - response = HttpResponseNotAuthorized() - return response + + return HttpResponseNotAuthorized() def basic_http_auth(func): + """A basic authentication decorator.""" + @wraps(func) - def inner(request, *args, **kwargs): + def _inner(request, *args, **kwargs): result = helper_auth_helper(request) if result is not None: return result return func(request, *args, **kwargs) - return inner + return _inner def http_auth_string(username, password): - credentials = base64.b64encode(( - '%s:%s' % (username, password)).encode('utf-8') - ).decode('utf-8').strip() - auth_string = 'Basic %s' % credentials + """Return a basic authentication string with username and password.""" + credentials = ( + base64.b64encode(f"{username}:{password}".encode("utf-8")) + .decode("utf-8") + .strip() + ) + auth_string = f"Basic {credentials}" + return auth_string def add_cors_headers(response): - response['Access-Control-Allow-Origin'] = '*' - response['Access-Control-Allow-Methods'] = 'GET' - response['Access-Control-Allow-Headers'] = ('Accept, Origin,' - ' X-Requested-With,' - ' Authorization') - response['Content-Type'] = 'application/json' + """Add CORS headers to the HttpResponse ``response`` instance.""" + response["Access-Control-Allow-Origin"] = "*" + response["Access-Control-Allow-Methods"] = "GET" + response[ + "Access-Control-Allow-Headers" + ] = "Accept, Origin, X-Requested-With, Authorization" + response["Content-Type"] = "application/json" + return response def set_api_permissions_for_user(user): - models = [ - UserProfile, XForm, MergedXForm, Project, Team, OrganizationProfile, - Note - ] + """Sets the permissions to allow a ``user`` to access the APU.""" + models = [UserProfile, XForm, MergedXForm, Project, Team, OrganizationProfile, Note] for model in models: for perm in get_perms_for_model(model): - assign_perm('%s.%s' % (perm.content_type.app_label, perm.codename), - user) + assign_perm(f"{perm.content_type.app_label}.{perm.codename}", user) def get_user_default_project(user): - name = u"{}'s Project".format(user.username) + """Return's the ``user``'s default project, creates it if it does not exist.'""" + name = f"{user.username}'s Project" user_projects = user.project_owner.filter(name=name, organization=user) if user_projects: project = user_projects[0] else: - metadata = {'description': 'Default Project'} + metadata = {"description": "Default Project"} project = Project.objects.create( - name=name, organization=user, created_by=user, metadata=metadata) + name=name, organization=user, created_by=user, metadata=metadata + ) return project @@ -230,7 +255,5 @@ def invalidate_and_regen_tokens(user): access_token = Token.objects.create(user=user).key temp_token = TempToken.objects.create(user=user).key - return { - 'access_token': access_token, - 'temp_token': temp_token - } + + return {"access_token": access_token, "temp_token": temp_token} From 7c56e9ea345cd3b4b2e4b04de23c8225391e95f0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 21:57:13 +0300 Subject: [PATCH 171/234] main.views: cleanup fixes --- onadata/apps/main/views.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 3d2710ed86..4014ce18e6 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -8,12 +8,10 @@ from datetime import datetime from http import HTTPStatus -from bson import json_util -from bson.objectid import ObjectId from django.conf import settings from django.contrib import messages -from django.contrib.auth.decorators import login_required from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required from django.core.cache import cache from django.core.files.storage import default_storage, get_storage_class from django.db import IntegrityError @@ -28,9 +26,12 @@ from django.shortcuts import get_object_or_404, render from django.template import loader from django.urls import reverse -from django.utils.translation import gettext as _ from django.utils.html import conditional_escape +from django.utils.translation import gettext as _ from django.views.decorators.http import require_GET, require_http_methods, require_POST + +from bson import json_util +from bson.objectid import ObjectId from guardian.shortcuts import assign_perm, remove_perm from oauth2_provider.views.base import AuthorizationView from rest_framework.authtoken.models import Token @@ -346,6 +347,9 @@ def profile_settings(request, username): if request.user.username != username: return HttpResponseNotFound("Page not found") content_user = check_and_set_user(request, username) + if isinstance(content_user, str): + return HttpResponseRedirect(content_user) + user_profile, _created = UserProfile.objects.get_or_create(user=content_user) if request.method == "POST": form = UserProfileForm(request.POST, instance=user_profile) @@ -382,8 +386,9 @@ def profile_settings(request, username): def public_profile(request, username): """Show user's public profile page view.""" content_user = check_and_set_user(request, username) - if isinstance(content_user, HttpResponseRedirect): - return content_user + if isinstance(content_user, str): + return HttpResponseRedirect(content_user) + data = {} set_profile_data(data, content_user) data["is_owner"] = request.user == content_user @@ -536,7 +541,7 @@ def api_token(request, username=None): # pylint: disable=too-many-locals,too-many-branches @require_http_methods(["GET", "OPTIONS"]) -def api(request, username=None, id_string=None): +def api(request, username=None, id_string=None): # noqa C901 """ Returns all results as JSON. If a parameter string is passed, it takes the 'query' parameter, converts this string to a dictionary, an @@ -649,7 +654,7 @@ def public_api(request, username, id_string): # pylint: disable=too-many-locals,too-many-branches,too-many-statements @login_required -def edit(request, username, id_string): +def edit(request, username, id_string): # noqa C901 """Edit form page view.""" xform = XForm.objects.get( user__username__iexact=username, @@ -1195,7 +1200,7 @@ def form_photos(request, username, id_string): # pylint: disable=too-many-branches @require_POST -def set_perm(request, username, id_string): +def set_perm(request, username, id_string): # noqa C901 """Assign form permissions to a user.""" xform = get_form( {"user__username__iexact": username, "id_string__iexact": id_string} From 83be5f14bdc038a7d07791b10f57dc017443d364 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 22:05:02 +0300 Subject: [PATCH 172/234] get_accounts_with_duplicate_id_strings.py: cleanup --- .../get_accounts_with_duplicate_id_strings.py | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py index 3f261f3137..b13b8cdd2e 100644 --- a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py +++ b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py @@ -1,41 +1,58 @@ +# -*- coding=utf-8 -*- +""" +get_accounts_with_duplicate_id_strings - Retrieves accounts with duplicate id_strings +""" +from pprint import pprint + from django.core.management.base import BaseCommand -from onadata.apps.logger.models.xform import XForm -from django.utils.translation import gettext_lazy from django.db.models import Count -from pprint import pprint +from django.utils.translation import gettext_lazy + +from onadata.apps.logger.models.xform import XForm class Command(BaseCommand): + """Retrieves accounts with duplicate id_strings""" + help = gettext_lazy("Retrieves accounts with duplicate id_strings") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): - duplicates = XForm.objects.values( - 'id_string', 'user__username').annotate( - id_string_count=Count('id_string')).filter(id_string_count__gt=1) + """Retrieves accounts with duplicate id_strings""" + duplicates = ( + XForm.objects.values("id_string", "user__username") + .annotate(id_string_count=Count("id_string")) + .filter(id_string_count__gt=1) + ) duplicates_dict = {} if len(duplicates) > 0: - for a in duplicates: + for dupe in duplicates: xforms = XForm.objects.filter( - id_string=a.get('id_string'), - user__username=a.get('user__username')) + id_string=dupe.get("id_string"), + user__username=dupe.get("user__username"), + ) for xform in xforms: if duplicates_dict.get(xform.user) is None: - duplicates_dict[xform.user] = [{ - "form id": xform.pk, - "no. of submission": xform.num_of_submissions, - "email": xform.user.email, - "id_string": xform.id_string - }] + duplicates_dict[xform.user] = [ + { + "form id": xform.pk, + "no. of submission": xform.num_of_submissions, + "email": xform.user.email, + "id_string": xform.id_string, + } + ] else: - duplicates_dict[xform.user].append({ - "form id": xform.pk, - "no. of submission": xform.num_of_submissions, - "email": xform.user.email, - "id_string": xform.id_string - }) + duplicates_dict[xform.user].append( + { + "form id": xform.pk, + "no. of submission": xform.num_of_submissions, + "email": xform.user.email, + "id_string": xform.id_string, + } + ) self.stdout.write( - 'Accounts with duplicate id_strings: %s' % - len(duplicates_dict)) + f"Accounts with duplicate id_strings: {len(duplicates_dict)}" + ) pprint(duplicates_dict) else: - self.stdout.write('Each account has a unique id_string :)') + self.stdout.write("Each account has a unique id_string :)") From cb837759f4088aa1ee47ff71bca71dcd84255fdb Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 22:25:26 +0300 Subject: [PATCH 173/234] batch: cleanup --- .../management/commands/reapplyperms.py | 54 ++++++++++------- .../management/commands/update_enketo_urls.py | 2 - onadata/libs/models/textit_service.py | 60 +++++++++++-------- onadata/libs/utils/chart_tools.py | 2 +- 4 files changed, 68 insertions(+), 50 deletions(-) diff --git a/onadata/apps/logger/management/commands/reapplyperms.py b/onadata/apps/logger/management/commands/reapplyperms.py index 1dcc9feebb..a38764ee70 100644 --- a/onadata/apps/logger/management/commands/reapplyperms.py +++ b/onadata/apps/logger/management/commands/reapplyperms.py @@ -1,23 +1,29 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +""" +reapplyperms - reapplies XForm permissions for the given user. +""" import gc - -from django.utils import timezone from datetime import timedelta -from onadata.apps.logger.models import XForm -from django.db.models import Q + from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from multidb.pinning import use_master -from onadata.libs.utils.project_utils import set_project_perms_to_xform +from onadata.apps.logger.models import XForm from onadata.libs.utils.model_tools import queryset_iterator +from onadata.libs.utils.project_utils import set_project_perms_to_xform DEFAULT_SECONDS = 30 * 60 class Command(BaseCommand): + """Reapply permissions to XForms.""" + help = _("Reapply permissions to XForms.") def _reapply_perms(self, username=None, days=None, seconds=None): @@ -35,31 +41,37 @@ def _reapply_perms(self, username=None, days=None, seconds=None): if the_past: xforms = XForm.objects.filter( - Q(date_created__gte=the_past) | - Q(date_modified__gte=the_past)) + Q(date_created__gte=the_past) | Q(date_modified__gte=the_past) + ) - self.stdout.write(_("{} to be updated").format(xforms.count())) + self.stdout.write(_(f"{xforms.count()} to be updated")) with use_master: for xform in queryset_iterator(xforms): set_project_perms_to_xform(xform, xform.project) - self.stdout.write( - _(f"Updated permissions for XForm ID: {xform.id}")) + self.stdout.write(_(f"Updated permissions for XForm ID: {xform.id}")) gc.collect() def add_arguments(self, parser): - parser.add_argument('--days', dest='days', type=int, default=0, - help=_("No of days")) - parser.add_argument('--seconds', dest='seconds', type=int, - default=DEFAULT_SECONDS, help=_("No of seconds")) - parser.add_argument('--username', dest='username', default=None, - help=_("Username")) + parser.add_argument( + "--days", dest="days", type=int, default=0, help=_("No of days") + ) + parser.add_argument( + "--seconds", + dest="seconds", + type=int, + default=DEFAULT_SECONDS, + help=_("No of seconds"), + ) + parser.add_argument( + "--username", dest="username", default=None, help=_("Username") + ) + # pylint: disable=unused-argument def handle(self, *args, **options): - days = int(options['days']) if 'days' in options else 0 - seconds = int(options['seconds']) if 'seconds' in options else \ - DEFAULT_SECONDS - username = options['username'] if 'username' in options else None + days = int(options["days"]) if "days" in options else 0 + seconds = int(options["seconds"]) if "seconds" in options else DEFAULT_SECONDS + username = options["username"] if "username" in options else None if username: self._reapply_perms(username=str(username)) diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 42e61b3b52..135b826187 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -78,8 +78,6 @@ def handle(self, *args, **options): data_value = meta_data.data_value xform = meta_data.content_object xform_pk = xform.pk - if not os.path.exists("/tmp/enketo_url"): - break form_url = get_form_url( request, diff --git a/onadata/libs/models/textit_service.py b/onadata/libs/models/textit_service.py index f04fec2fd8..10e03264b0 100644 --- a/onadata/libs/models/textit_service.py +++ b/onadata/libs/models/textit_service.py @@ -3,28 +3,38 @@ TextItService model: sets up all properties for interaction with TextIt or RapidPro. """ -from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.restservice.models import RestService from django.conf import settings -from django.utils.translation import gettext as _ from django.db import IntegrityError +from django.utils.translation import gettext as _ + from rest_framework import serializers +from onadata.apps.main.models.meta_data import MetaData +from onadata.apps.restservice.models import RestService + METADATA_SEPARATOR = settings.METADATA_SEPARATOR -# pylint: disable=R0902 -class TextItService(object): +# pylint: disable=too-many-instance-attributes +class TextItService: """ TextItService model: access/create/update RestService and MetaData objects with all properties for TextIt or RapidPro like services. """ - # pylint: disable=R0913 - def __init__(self, xform, service_url=None, name=None, auth_token=None, - flow_uuid=None, contacts=None, - pk=None, flow_title: str = ""): - self.pk = pk # pylint: disable=C0103 + # pylint: disable=too-many-arguments,invalid-name + def __init__( + self, + xform, + service_url=None, + name=None, + auth_token=None, + flow_uuid=None, + contacts=None, + pk=None, + flow_title: str = "", + ): + self.pk = pk self.xform = xform self.auth_token = auth_token self.flow_uuid = flow_uuid @@ -42,8 +52,7 @@ def save(self): Creates and updates RestService and MetaData objects with textit or rapidpro service properties. """ - service = RestService() if not self.pk else \ - RestService.objects.get(pk=self.pk) + service = RestService() if not self.pk else RestService.objects.get(pk=self.pk) service.name = self.name service.service_url = self.service_url @@ -51,9 +60,8 @@ def save(self): try: service.save() except IntegrityError as e: - if str(e).startswith("duplicate key value violates unique " - "constraint"): - msg = _(u"The service already created for this form.") + if str(e).startswith("duplicate key value violates unique constraint"): + msg = _("The service already created for this form.") else: msg = _(str(e)) raise serializers.ValidationError(msg) @@ -64,13 +72,10 @@ def save(self): self.active = service.active self.inactive_reason = service.inactive_reason - data_value = '{}|{}|{}'.format(self.auth_token, - self.flow_uuid, - self.contacts) + data_value = f"{self.auth_token}|{self.flow_uuid}|{self.contacts}" MetaData.textit(self.xform, data_value=data_value) - MetaData.textit_flow_details( - self.xform, data_value=self.flow_title) + MetaData.textit_flow_details(self.xform, data_value=self.flow_title) if self.xform.is_merged_dataset: for xform in self.xform.mergedxform.xforms.all(): @@ -87,10 +92,13 @@ def retrieve(self): data_value = MetaData.textit(self.xform) try: - self.auth_token, self.flow_uuid, self.contacts = \ - data_value.split(METADATA_SEPARATOR) - except ValueError: + self.auth_token, self.flow_uuid, self.contacts = data_value.split( + METADATA_SEPARATOR + ) + except ValueError as e: raise serializers.ValidationError( - _("Error occurred when loading textit service." - "Resolve by updating auth_token, flow_uuid and contacts" - " fields")) + _( + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and contacts fields" + ) + ) from e diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index 046e0bc53c..a26796c627 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -225,7 +225,7 @@ def _use_labels_from_group_by_name( # noqa C901 # pylint: disable=too-many-locals,too-many-branches,too-many-arguments -def build_chart_data_for_field( # noqa +def build_chart_data_for_field( # noqa C901 xform, field, language_index=0, choices=None, group_by=None, data_view=None ): """Returns the chart data for a given field.""" From 1f04916eef77335afd36a6125d9b465e672ff912 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 22:52:40 +0300 Subject: [PATCH 174/234] Refactor create_attachments_zipfile() Use with statements when opening files. --- onadata/apps/viewer/views.py | 11 +- onadata/libs/tests/utils/test_viewer_tools.py | 109 ++++++++------- onadata/libs/utils/export_tools.py | 124 ++++++++---------- onadata/libs/utils/viewer_tools.py | 16 +-- 4 files changed, 127 insertions(+), 133 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 722d2b4e0b..cf3dd20f91 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -293,7 +293,7 @@ def thank_you_submission(request, username, id_string): # pylint: disable=too-many-locals -def data_export(request, username, id_string, export_type): +def data_export(request, username, id_string, export_type): # noqa C901 """ Data export view. """ @@ -481,7 +481,7 @@ def _get_google_credential(request): return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) -def export_list(request, username, id_string, export_type): +def export_list(request, username, id_string, export_type): # noqa C901 """ Export list view. """ @@ -714,8 +714,8 @@ def zip_export(request, username, id_string): attachments = Attachment.objects.filter(instance__xform=xform) zip_file = None - try: - zip_file = create_attachments_zipfile(attachments) + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(attachments, zip_file) audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT} audit_log( Actions.EXPORT_CREATED, @@ -741,9 +741,6 @@ def zip_export(request, username, id_string): response.write(FileWrapper(zip_file)) response["Content-Length"] = zip_file.tell() zip_file.seek(0) - finally: - if zip_file: - zip_file.close() return response diff --git a/onadata/libs/tests/utils/test_viewer_tools.py b/onadata/libs/tests/utils/test_viewer_tools.py index 9e0b492bda..2ed76c193f 100644 --- a/onadata/libs/tests/utils/test_viewer_tools.py +++ b/onadata/libs/tests/utils/test_viewer_tools.py @@ -3,19 +3,24 @@ import os from django.core.files.base import File +from django.core.files.temp import NamedTemporaryFile from django.http import Http404 from django.test.client import RequestFactory from django.test.utils import override_settings from django.utils import timezone + from mock import patch -from onadata.apps.logger.models import XForm, Instance, Attachment +from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.utils.viewer_tools import (create_attachments_zipfile, - export_def_from_filename, - generate_enketo_form_defaults, - get_client_ip, get_form, - get_form_url) +from onadata.libs.utils.viewer_tools import ( + create_attachments_zipfile, + export_def_from_filename, + generate_enketo_form_defaults, + get_client_ip, + get_form, + get_form_url, +) class TestViewerTools(TestBase): @@ -25,8 +30,8 @@ def test_export_def_from_filename(self): """Test export_def_from_filename().""" filename = "path/filename.xlsx" ext, mime_type = export_def_from_filename(filename) - self.assertEqual(ext, 'xlsx') - self.assertEqual(mime_type, 'vnd.openxmlformats') + self.assertEqual(ext, "xlsx") + self.assertEqual(mime_type, "vnd.openxmlformats") def test_get_client_ip(self): """Test get_client_ip().""" @@ -53,14 +58,12 @@ def test_get_enketo_defaults_with_right_xform(self): # create xform self._publish_transportation_form() # create kwargs with existing xform variable - xform_variable_name = \ - 'available_transportation_types_to_referral_facility' - xform_variable_value = 'ambulance' + xform_variable_name = "available_transportation_types_to_referral_facility" + xform_variable_value = "ambulance" kwargs = {xform_variable_name: xform_variable_value} defaults = generate_enketo_form_defaults(self.xform, **kwargs) - key = "defaults[/data/transport/{}]".format( - xform_variable_name) + key = "defaults[/data/transport/{}]".format(xform_variable_name) self.assertEqual(defaults, {key: xform_variable_value}) # pylint: disable=C0103 @@ -69,25 +72,26 @@ def test_get_enketo_defaults_with_multiple_params(self): # create xform self._publish_transportation_form() # create kwargs with existing xform variable - transportation_types = \ - 'available_transportation_types_to_referral_facility' - transportation_types_value = 'ambulance' + transportation_types = "available_transportation_types_to_referral_facility" + transportation_types_value = "ambulance" - frequency = 'frequency_to_referral_facility' - frequency_value = 'daily' + frequency = "frequency_to_referral_facility" + frequency_value = "daily" kwargs = { transportation_types: transportation_types_value, - frequency: frequency_value + frequency: frequency_value, } defaults = generate_enketo_form_defaults(self.xform, **kwargs) - transportation_types_key = \ - "defaults[/data/transport/{}]".format( - transportation_types) - frequency_key = "defaults[/data/transport/"\ - "loop_over_transport_types_frequency/"\ - "{}/{}]".format(transportation_types_value, frequency) + transportation_types_key = "defaults[/data/transport/{}]".format( + transportation_types + ) + frequency_key = ( + "defaults[/data/transport/" + "loop_over_transport_types_frequency/" + "{}/{}]".format(transportation_types_value, frequency) + ) self.assertIn(transportation_types_key, defaults) self.assertIn(frequency_key, defaults) @@ -97,7 +101,7 @@ def test_get_enketo_defaults_with_non_existent_field(self): # create xform self._publish_transportation_form() # create kwargs with NON-existing xform variable - kwargs = {'name': 'bla'} + kwargs = {"name": "bla"} defaults = generate_enketo_form_defaults(self.xform, **kwargs) self.assertEqual(defaults, {}) @@ -105,17 +109,17 @@ def test_get_form(self): """Test get_form().""" # non existent id_string with self.assertRaises(Http404): - get_form({'id_string': 'non_existent_form'}) + get_form({"id_string": "non_existent_form"}) self._publish_transportation_form() # valid xform id_string - kwarg = {'id_string__iexact': self.xform.id_string} + kwarg = {"id_string__iexact": self.xform.id_string} xform = get_form(kwarg) self.assertIsInstance(xform, XForm) # pass a queryset - kwarg['queryset'] = XForm.objects.all() + kwarg["queryset"] = XForm.objects.all() xform = get_form(kwarg) self.assertIsInstance(xform, XForm) @@ -128,36 +132,34 @@ def test_get_form(self): @override_settings(TESTING_MODE=False) def test_get_form_url(self): """Test get_form_url().""" - request = RequestFactory().get('/') + request = RequestFactory().get("/") # default https://ona.io url = get_form_url(request) - self.assertEqual(url, 'https://ona.io') + self.assertEqual(url, "https://ona.io") # with username https://ona.io/bob - url = get_form_url(request, username='bob') - self.assertEqual(url, 'https://ona.io/bob') + url = get_form_url(request, username="bob") + self.assertEqual(url, "https://ona.io/bob") # with http protocol http://ona.io/bob - url = get_form_url(request, username='bob', protocol='http') - self.assertEqual(url, 'http://ona.io/bob') + url = get_form_url(request, username="bob", protocol="http") + self.assertEqual(url, "http://ona.io/bob") # preview url http://ona.io/preview/bob - url = get_form_url( - request, username='bob', protocol='http', preview=True) - self.assertEqual(url, 'http://ona.io/preview/bob') + url = get_form_url(request, username="bob", protocol="http", preview=True) + self.assertEqual(url, "http://ona.io/preview/bob") # with form pk url http://ona.io/bob/1 - url = get_form_url(request, username='bob', xform_pk=1) - self.assertEqual(url, 'https://ona.io/bob/1') + url = get_form_url(request, username="bob", xform_pk=1) + self.assertEqual(url, "https://ona.io/bob/1") # with form uuid url https://ona.io/enketo/492 - url = get_form_url( - request, xform_pk=492, generate_consistent_urls=True) - self.assertEqual(url, 'https://ona.io/enketo/492') + url = get_form_url(request, xform_pk=492, generate_consistent_urls=True) + self.assertEqual(url, "https://ona.io/enketo/492") @override_settings(ZIP_REPORT_ATTACHMENT_LIMIT=8) - @patch('onadata.libs.utils.viewer_tools.report_exception') + @patch("onadata.libs.utils.viewer_tools.report_exception") def test_create_attachments_zipfile_file_too_big(self, rpt_mock): """ When a file is larger than what is allowed in settings an exception @@ -166,16 +168,21 @@ def test_create_attachments_zipfile_file_too_big(self, rpt_mock): self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create( - instance=self.instance, - media_file=File(open(media_file, 'rb'), media_file)) - create_attachments_zipfile(Attachment.objects.all()) + instance=self.instance, media_file=File(open(media_file, "rb"), media_file) + ) + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(Attachment.objects.all(), zip_file) - message = ( - "Create attachment zip exception", "File is greater than 8 bytes") + message = ("Create attachment zip exception", "File is greater than 8 bytes") self.assertTrue(rpt_mock.called) rpt_mock.assert_called_with(message[0], message[1]) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index eb11da3c75..55c30cdf34 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -11,10 +11,8 @@ import sys from datetime import datetime, timedelta -import builtins -import six from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.files.base import File from django.core.files.storage import default_storage from django.core.files.temp import NamedTemporaryFile @@ -22,18 +20,19 @@ from django.shortcuts import render from django.utils import timezone from django.utils.translation import gettext as _ -from six.moves.urllib.parse import urlparse -from six import iteritems + +import six from json2xlsclient.client import Client +from multidb.pinning import use_master from rest_framework import exceptions +from six import iteritems +from six.moves.urllib.parse import urlparse try: from savReaderWriter import SPSSIOError except ImportError: SPSSIOError = Exception -from multidb.pinning import use_master - from onadata.apps.logger.models import Attachment, Instance, OsmData, XForm from onadata.apps.logger.models.data_view import DataView from onadata.apps.main.models.meta_data import MetaData @@ -42,10 +41,10 @@ from onadata.libs.exceptions import J2XException, NoRecordsFoundError from onadata.libs.utils.common_tags import DATAVIEW_EXPORT, GROUPNAME_REMOVED_FLAG from onadata.libs.utils.common_tools import ( - str_to_bool, cmp_to_key, report_exception, retry, + str_to_bool, ) from onadata.libs.utils.export_builder import ExportBuilder from onadata.libs.utils.model_tools import get_columns_with_hxl, queryset_iterator @@ -58,6 +57,9 @@ EXPORT_QUERY_KEY = "query" MAX_RETRIES = 3 +# pylint: disable=invalid-name +User = get_user_model() + def md5hash(string): """ @@ -90,8 +92,6 @@ def get_or_create_export(export_id, xform, export_type, options): return Export.objects.get(pk=export_id) except Export.DoesNotExist: if getattr(settings, "SLAVE_DATABASES", []): - from multidb.pinning import use_master - with use_master: try: return Export.objects.get(pk=export_id) @@ -103,7 +103,7 @@ def get_or_create_export(export_id, xform, export_type, options): # pylint: disable=too-many-locals, too-many-branches, too-many-statements @retry(MAX_RETRIES) -def generate_export(export_type, xform, export_id=None, options=None): +def generate_export(export_type, xform, export_id=None, options=None): # noqa C901 """ Create appropriate export object given the export type. @@ -208,6 +208,7 @@ def generate_export(export_type, xform, export_id=None, options=None): # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) + # pylint: disable=broad-except try: func.__call__( temp_file.name, @@ -234,13 +235,13 @@ def generate_export(export_type, xform, export_id=None, options=None): return export # generate filename - basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) + basename = f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")}' if remove_group_name: # add 'remove group name' flag to filename - basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) + basename = f"{basename}-{GROUPNAME_REMOVED_FLAG}" if dataview: - basename = "{}-{}".format(basename, DATAVIEW_EXPORT) + basename = f"{basename}-{DATAVIEW_EXPORT}" filename = basename + "." + extension @@ -306,7 +307,7 @@ def check_pending_export( export_type=export_type, internal_status=Export.PENDING, created_on__gt=created_time, - **export_options_kwargs + **export_options_kwargs, ).last() return export @@ -389,11 +390,12 @@ def increment_index_in_filename(filename): index = 1 # split filename from ext basename, ext = os.path.splitext(filename) - new_filename = "%s-%d%s" % (basename, index, ext) + new_filename = f"{basename}-{index}{ext}" + return new_filename -# pylint: disable=R0913 +# pylint: disable=too-many-arguments def generate_attachments_zip_export( export_type, username, id_string, export_id=None, options=None, xform=None ): @@ -429,37 +431,28 @@ def generate_attachments_zip_export( attachments = Attachment.objects.filter(instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( - instance__xform_id__in=[ - i - for i in xform.mergedxform.xforms.filter( + instance__xform_id__in=list( + xform.mergedxform.xforms.filter( deleted_at__isnull=True ).values_list("id", flat=True) - ] + ) ).filter(instance_id__in=[i_id["_id"] for i_id in instance_ids]) else: attachments = attachments.filter(instance__xform_id=xform.pk).filter( instance_id__in=[i_id["_id"] for i_id in instance_ids] ) - filename = "%s_%s.%s" % ( - id_string, - datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), - export_type.lower(), + filename = ( + f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' + f".{export_type.lower()}" ) file_path = os.path.join(username, "exports", id_string, export_type, filename) zip_file = None - try: - zip_file = create_attachments_zipfile(attachments) - - try: - temp_file = builtins.open(zip_file.name, "rb") + with NamedTemporaryFile() as zip_file: + create_attachments_zipfile(attachments, zip_file) + with open(zip_file.name, "rb") as temp_file: filename = default_storage.save(file_path, File(temp_file, file_path)) - finally: - temp_file.close() - finally: - if zip_file: - zip_file.close() export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) @@ -512,7 +505,7 @@ def get_or_create_export_object(export_id, options, xform, export_type): return export -# pylint: disable=R0913 +# pylint: disable=too-many-arguments def generate_kml_export( export_type, username, id_string, export_id=None, options=None, xform=None ): @@ -536,7 +529,7 @@ def generate_kml_export( None, "survey.kml", {"data": kml_export_data(id_string, user, xform=xform)} ) - basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + basename = f'{id_string}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' filename = basename + "." + export_type.lower() file_path = os.path.join(username, "exports", id_string, export_type, filename) @@ -574,12 +567,11 @@ def cached_get_labels(xpath): if xform.is_merged_dataset: data_kwargs.update( { - "xform_id__in": [ - i - for i in xform.mergedxform.xforms.filter( + "xform_id__in": list( + xform.mergedxform.xforms.filter( deleted_at__isnull=True ).values_list("id", flat=True) - ] + ) } ) else: @@ -593,14 +585,16 @@ def cached_get_labels(xpath): xpaths = list(data_for_display) xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp())) table_rows = [ - "%s%s" - % (cached_get_labels(xpath), data_for_display[xpath]) + f"{cached_get_labels(xpath) }" + f"{data_for_display[xpath]}" for xpath in xpaths if not xpath.startswith("_") ] img_urls = image_urls(instance) if instance.point: + img_url = img_urls[0] if img_urls else "" + rows = "".join(table_rows) data_for_template.append( { "name": instance.xform.id_string, @@ -609,8 +603,8 @@ def cached_get_labels(xpath): "lng": instance.point.x, "image_urls": img_urls, "table": '%s' - "
    " % (img_urls[0] if img_urls else "", "".join(table_rows)), + f'class="thumbnail" src="{img_url}" alt="">{rows}' + "", } ) @@ -623,12 +617,11 @@ def get_osm_data_kwargs(xform): kwargs = {"instance__deleted_at__isnull": True} if xform.is_merged_dataset: - kwargs["instance__xform_id__in"] = [ - i - for i in xform.mergedxform.xforms.filter( - deleted_at__isnull=True - ).values_list("id", flat=True) - ] + kwargs["instance__xform_id__in"] = list( + xform.mergedxform.xforms.filter(deleted_at__isnull=True).values_list( + "id", flat=True + ) + ) else: kwargs["instance__xform_id"] = xform.pk @@ -657,8 +650,8 @@ def generate_osm_export( kwargs = get_osm_data_kwargs(xform) osm_list = OsmData.objects.filter(**kwargs) content = get_combined_osm(osm_list) - - basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + basename = f"{id_string}_{timestamp}" filename = basename + "." + extension file_path = os.path.join(username, "exports", id_string, export_type, filename) @@ -704,16 +697,14 @@ def _get_server_from_metadata(xform, meta, token): try: report_templates = MetaData.external_export(xform) except MetaData.DoesNotExist: - from multidb.pinning import use_master - with use_master: report_templates = MetaData.external_export(xform) if meta: try: int(meta) - except ValueError: - raise Exception("Invalid metadata pk {0}".format(meta)) + except ValueError as e: + raise Exception(f"Invalid metadata pk {meta}") from e # Get the external server from the metadata result = report_templates.get(pk=meta) @@ -735,7 +726,7 @@ def _get_server_from_metadata(xform, meta, token): return server, name -def generate_external_export( +def generate_external_export( # noqa C901 export_type, username, id_string, export_id=None, options=None, xform=None ): """ @@ -794,14 +785,14 @@ def generate_external_export( status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( - "J2X client could not generate report. Server -> {0}," - " Error-> {1}".format(server, e) - ) + f"J2X client could not generate report. Server -> {server}," + f" Error-> {e}" + ) from e else: if not server: raise J2XException("External server not set") - elif not records: - raise J2XException("No record to export. Form -> {0}".format(id_string)) + if not records: + raise J2XException(f"No record to export. Form -> {id_string}") # get or create export object if export_id: @@ -846,7 +837,8 @@ def upload_template_for_external_export(server, file_obj): return str(status_code) + "|" + response -def parse_request_export_options(params): # pylint: disable=too-many-branches +# pylint: disable=too-many-branches +def parse_request_export_options(params): # noqa C901 """ Parse export options in the request object into values returned in a list. The list represents a boolean for whether the group name should be @@ -944,8 +936,8 @@ def get_repeat_index_tags(index_tags): if tag not in SUPPORTED_INDEX_TAGS: raise exceptions.ParseError( _( - "The tag %s is not supported, supported tags are %s" - % (tag, SUPPORTED_INDEX_TAGS) + f"The tag {tag} is not supported, " + f"supported tags are {SUPPORTED_INDEX_TAGS}" ) ) diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index d86c7a34f4..1249ee8bab 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -5,7 +5,6 @@ import sys import zipfile from json.decoder import JSONDecodeError -from tempfile import NamedTemporaryFile from typing import Dict from defusedxml import minidom @@ -232,12 +231,13 @@ def generate_enketo_form_defaults(xform, **kwargs): return defaults -def create_attachments_zipfile(attachments): - """Return a zip file with submission attachments.""" - # create zip_file - # pylint: disable=consider-using-with - tmp = NamedTemporaryFile() - with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: +def create_attachments_zipfile(attachments, zip_file): + """Return a zip file with submission attachments. + + :param attachments: an Attachments queryset. + :param zip_file: a file object, more likely a NamedTemporaryFile() object. + """ + with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name @@ -259,8 +259,6 @@ def create_attachments_zipfile(attachments): report_exception("Create attachment zip exception", e) break - return tmp - def get_form(kwargs): """Return XForm object by applying kwargs on an XForm queryset.""" From 7ca8fe15ba02f75dae57a32fd9bcf1020603286a Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 7 May 2022 23:51:55 +0300 Subject: [PATCH 175/234] twilio.py: cleanup --- onadata/apps/sms_support/providers/twilio.py | 125 +++++++++++-------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/onadata/apps/sms_support/providers/twilio.py b/onadata/apps/sms_support/providers/twilio.py index 79bde0b75e..d2eeb2b2c7 100644 --- a/onadata/apps/sms_support/providers/twilio.py +++ b/onadata/apps/sms_support/providers/twilio.py @@ -11,71 +11,80 @@ import datetime -from dict2xml import dict2xml from django.http import HttpResponse from django.urls import reverse from django.utils.translation import gettext as _ -from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +from dict2xml import dict2xml -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'twilio'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'twilio'}) - doc = (u'

    ' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Twilio\'s SMS Request'} + - u'

    1. ' + - _(u"Sign in to Twilio.com and go your Application.") + - u'
    2. ' + - _(u"Follow instructions to add one of the following URLs, " - u"selecting the HTTP POST method:") + - u'
      %(urla)s' + - u'
      %(urlb)s
      ' + - u'

    ' + - _(u"That's it. Now Send an SMS Formhub submission to your Twilio" - u" phone number. It will create a submission on Formhub.") + - u'

    ') % {'urla': urla, 'urlb': urlb} + """Returns Twilio integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "twilio"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "twilio"}, + ) + doc = ( + "

    " + + _("%(service)s Instructions:") + % {"service": '' "Twilio's SMS Request"} + + "

    1. " + + _("Sign in to Twilio.com and go your Application.") + + "
    2. " + + _( + "Follow instructions to add one of the following URLs, " + "selecting the HTTP POST method:" + ) + + '
      %(urla)s' + + "
      %(urlb)s
      " + + "

    " + + _( + "That's it. Now Send an SMS Formhub submission to your Twilio" + " phone number. It will create a submission on Formhub." + ) + + "

    " + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Return an XML formatted HttpResponse based on the ``data`` provided.""" - xml_head = u'' - response_dict = {'Response': {}} - message = data.get('text') + xml_head = '' + response_dict = {"Response": {}} + message = data.get("text") - if data.get('code') == SMS_API_ERROR: + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _("[ERROR] %s") % message if message: - messages = [message, ] - sendouts = data.get('sendouts', []) + messages = [ + message, + ] + sendouts = data.get("sendouts", []) if len(sendouts): messages += sendouts - response_dict.update({"Response": {'Sms': messages}}) + response_dict.update({"Response": {"Sms": messages}}) response = xml_head + dict2xml(response_dict) - return HttpResponse(response, content_type='text/xml') + return HttpResponse(response, content_type="text/xml") @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @@ -83,33 +92,41 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('From', '').strip() - sms_text = request.POST.get('Body', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('time_created', now_timestamp).strip() + sms_identity = request.POST.get("From", "").strip() + sms_text = request.POST.get("Body", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("time_created", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_twilio(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_twilio( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_twilio(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument +def process_message_for_twilio(username, sms_identity, sms_text, sms_time, id_string): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] From 6aaccbab920b676848e303af466cd9f9f055b95b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 00:08:58 +0300 Subject: [PATCH 176/234] telerivet.py: cleanup --- .../management/commands/update_enketo_urls.py | 1 - .../apps/sms_support/providers/telerivet.py | 112 ++++++++++-------- 2 files changed, 63 insertions(+), 50 deletions(-) diff --git a/onadata/apps/main/management/commands/update_enketo_urls.py b/onadata/apps/main/management/commands/update_enketo_urls.py index 135b826187..89f18cb3c2 100644 --- a/onadata/apps/main/management/commands/update_enketo_urls.py +++ b/onadata/apps/main/management/commands/update_enketo_urls.py @@ -3,7 +3,6 @@ update_enketo_urls - command to update Enketo preview URLs in the MetaData model. """ import argparse -import os import sys from django.core.management.base import BaseCommand, CommandError diff --git a/onadata/apps/sms_support/providers/telerivet.py b/onadata/apps/sms_support/providers/telerivet.py index 5bf25030f0..69213ca16c 100644 --- a/onadata/apps/sms_support/providers/telerivet.py +++ b/onadata/apps/sms_support/providers/telerivet.py @@ -4,70 +4,74 @@ See: http://telerivet.com/help/api/webhook/receiving """ -import json import datetime -from django.http import HttpResponse +from django.http import JsonResponse from django.urls import reverse from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'telerivet'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'telerivet'}) - doc = (u'

    ' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Telerivet\'s Webhook API'} + - u'

    1. ' + - _(u"Sign in to Telerivet.com and go to Service Page.") + - u'
    2. ' + - _(u"Follow instructions to add an application with either URL:") + - u'
      %(urla)s' + - u'
      %(urlb)s
      ' + - u'

    ' + - _(u"That's it. Now Send an SMS Formhub submission to your Telerivet" - u" phone number. It will create a submission on Formhub.") + - u'

    ') % {'urla': urla, 'urlb': urlb} + """Returns Telerivet integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "telerivet"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "telerivet"}, + ) + doc = ( + "

    " + + _("%(service)s Instructions:") + % {"service": '' "Telerivet's Webhook API"} + + "

    1. " + + _("Sign in to Telerivet.com and go to Service Page.") + + "
    2. " + + _("Follow instructions to add an application with either URL:") + + '
      %(urla)s' + + "
      %(urlb)s
      " + + "

    " + + _( + "That's it. Now Send an SMS Formhub submission to your Telerivet" + " phone number. It will create a submission on Formhub." + ) + + "

    " + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Return a JSON formatted HttpResponse based on the ``data`` provided.""" - message = data.get('text') + message = data.get("text") - if data.get('code') == SMS_API_ERROR: + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _(f"[ERROR] {message}") response = {} if message: messages = [{"content": message}] - sendouts = data.get('sendouts', []) + sendouts = data.get("sendouts", []) if len(sendouts): messages += [{"content": text} for text in sendouts] response.update({"messages": messages}) - return HttpResponse(json.dumps(response), content_type='application/json') + + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @@ -75,33 +79,43 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('from_number', '').strip() - sms_text = request.POST.get('content', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('time_created', now_timestamp).strip() + sms_identity = request.POST.get("from_number", "").strip() + sms_text = request.POST.get("content", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("time_created", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_telerivet(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_telerivet( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_telerivet(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument +def process_message_for_telerivet( + username, sms_identity, sms_text, sms_time, id_string +): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] From 0a7faae4a1b95ca281ac1d701773dcadd119b6f6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 00:13:25 +0300 Subject: [PATCH 177/234] smssync.py: cleanup --- onadata/apps/sms_support/providers/smssync.py | 139 ++++++++++-------- 1 file changed, 75 insertions(+), 64 deletions(-) diff --git a/onadata/apps/sms_support/providers/smssync.py b/onadata/apps/sms_support/providers/smssync.py index b7bd71a781..bff0a81e1d 100644 --- a/onadata/apps/sms_support/providers/smssync.py +++ b/onadata/apps/sms_support/providers/smssync.py @@ -7,117 +7,128 @@ See: http://smssync.ushahidi.com/doc """ -import json import datetime -from django.http import HttpResponse +from django.http import JsonResponse from django.urls import reverse from django.utils.translation import gettext as _ -from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'smssync'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'smssync'}) - doc = (u'

    ' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'Ushaidi\'s SMS Sync'} + - u'

    1. ' + - _(u"Download the SMS Sync App on your phone serving " - u"as a gateway.") + '
    2. ' + - _(u"Configure the app to point to one of the following URLs") + - u'
      %(urla)s' + - u'
      %(urlb)s
      ' + - _(u"Optionnaly set a keyword to prevent non-formhub " - u"messages to be sent.") + - '
    3. ' + - _(u"In the preferences, tick the box to allow " - u"replies from the server.") + - '

    ' + - _(u"That's it. Now Send an SMS Formhub submission to the number " - u"of that phone. It will create a submission on Formhub.") + - u'

    ') % {'urla': urla, 'urlb': urlb} + """Returns SMSSync integration documentation.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "smssync"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "smssync"}, + ) + doc = ( + "

    " + + _("%(service)s Instructions:") + % { + "service": '' + "Ushaidi's SMS Sync" + } + + "

    1. " + + _("Download the SMS Sync App on your phone serving " "as a gateway.") + + "
    2. " + + _("Configure the app to point to one of the following URLs") + + '
      %(urla)s' + + "
      %(urlb)s
      " + + _("Optionnaly set a keyword to prevent non-formhub " "messages to be sent.") + + "
    3. " + + _("In the preferences, tick the box to allow " "replies from the server.") + + "

    " + + _( + "That's it. Now Send an SMS Formhub submission to the number " + "of that phone. It will create a submission on Formhub." + ) + + "

    " + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): - message = data.get('text') - if data.get('code') == SMS_API_ERROR: + """Return a JSON formatted HttpResponse based on the ``data`` provided.""" + message = data.get("text") + if data.get("code") == SMS_API_ERROR: success = False message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: success = True - message = _(u"[ERROR] %s") % message + message = _("[ERROR] %s") % message else: success = True - response = { - "payload": { - "success": success, - "task": "send"}} + response = {"payload": {"success": success, "task": "send"}} if message: - messages = [{"to": data.get('identity'), "message": message}] - sendouts = data.get('sendouts', []) + messages = [{"to": data.get("identity"), "message": message}] + sendouts = data.get("sendouts", []) if len(sendouts): - messages += [{"to": data.get('identity'), "message": text} - for text in sendouts] - response['payload'].update({"messages": messages}) - return HttpResponse(json.dumps(response), content_type='application/json') + messages += [ + {"to": data.get("identity"), "message": text} for text in sendouts + ] + response["payload"].update({"messages": messages}) + + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_identity = request.POST.get('from', '').strip() - sms_text = request.POST.get('message', '').strip() - now_timestamp = datetime.datetime.now().strftime('%s') - sent_timestamp = request.POST.get('sent_timestamp', now_timestamp).strip() + sms_identity = request.POST.get("from", "").strip() + sms_text = request.POST.get("message", "").strip() + now_timestamp = datetime.datetime.now().strftime("%s") + sent_timestamp = request.POST.get("sent_timestamp", now_timestamp).strip() try: sms_time = datetime.datetime.fromtimestamp(float(sent_timestamp)) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_smssync(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string) + return process_message_for_smssync( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + ) -def process_message_for_smssync(username, - sms_identity, sms_text, sms_time, id_string): - """ Process a text instance and return in SMSSync expected format """ +# pylin: disable=unused-argument +def process_message_for_smssync(username, sms_identity, sms_text, sms_time, id_string): + """Process a text instance and return in SMSSync expected format""" if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] - response.update({'identity': sms_identity}) + response.update({"identity": sms_identity}) return get_response(response) From f956848b12d90e7e757528465291ca0d448d1dbc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 00:17:00 +0300 Subject: [PATCH 178/234] sms_support.providers: cleanup --- .../apps/sms_support/providers/__init__.py | 132 +++++++++++------- 1 file changed, 78 insertions(+), 54 deletions(-) diff --git a/onadata/apps/sms_support/providers/__init__.py b/onadata/apps/sms_support/providers/__init__.py index fa353ade61..09b01f9549 100644 --- a/onadata/apps/sms_support/providers/__init__.py +++ b/onadata/apps/sms_support/providers/__init__.py @@ -1,82 +1,106 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support.providers +""" from __future__ import absolute_import from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt -from onadata.apps.sms_support.providers.smssync import \ - autodoc as autodoc_smssync -from onadata.apps.sms_support.providers.smssync import \ - import_submission as imp_sub_smssync -from onadata.apps.sms_support.providers.smssync import \ - import_submission_for_form as imp_sub_form_smssync -from onadata.apps.sms_support.providers.telerivet import \ - autodoc as autodoc_telerivet -from onadata.apps.sms_support.providers.telerivet import \ - import_submission as imp_sub_telerivet -from onadata.apps.sms_support.providers.telerivet import \ - import_submission_for_form as imp_sub_form_telerivet +from onadata.apps.sms_support.providers.smssync import autodoc as autodoc_smssync +from onadata.apps.sms_support.providers.smssync import ( + import_submission as imp_sub_smssync, +) +from onadata.apps.sms_support.providers.smssync import ( + import_submission_for_form as imp_sub_form_smssync, +) +from onadata.apps.sms_support.providers.telerivet import autodoc as autodoc_telerivet +from onadata.apps.sms_support.providers.telerivet import ( + import_submission as imp_sub_telerivet, +) +from onadata.apps.sms_support.providers.telerivet import ( + import_submission_for_form as imp_sub_form_telerivet, +) from onadata.apps.sms_support.providers.textit import autodoc as autodoc_textit -from onadata.apps.sms_support.providers.textit import \ - import_submission as imp_sub_textit -from onadata.apps.sms_support.providers.textit import \ - import_submission_for_form as imp_sub_form_textit +from onadata.apps.sms_support.providers.textit import ( + import_submission as imp_sub_textit, +) +from onadata.apps.sms_support.providers.textit import ( + import_submission_for_form as imp_sub_form_textit, +) from onadata.apps.sms_support.providers.twilio import autodoc as autodoc_twilio -from onadata.apps.sms_support.providers.twilio import \ - import_submission as imp_sub_twilio -from onadata.apps.sms_support.providers.twilio import \ - import_submission_for_form as imp_sub_form_twilio +from onadata.apps.sms_support.providers.twilio import ( + import_submission as imp_sub_twilio, +) +from onadata.apps.sms_support.providers.twilio import ( + import_submission_for_form as imp_sub_form_twilio, +) -SMSSYNC = 'smssync' -TELERIVET = 'telerivet' -TWILIO = 'twilio' -TEXTIT = 'textit' +SMSSYNC = "smssync" +TELERIVET = "telerivet" +TWILIO = "twilio" +TEXTIT = "textit" PROVIDERS = { - SMSSYNC: {'name': u"SMS Sync", - 'imp': imp_sub_smssync, - 'imp_form': imp_sub_form_smssync, - 'doc': autodoc_smssync}, - TELERIVET: {'name': u"Telerivet", - 'imp': imp_sub_telerivet, - 'imp_form': imp_sub_form_telerivet, - 'doc': autodoc_telerivet}, - TWILIO: {'name': u"Twilio", - 'imp': imp_sub_twilio, - 'imp_form': imp_sub_form_twilio, - 'doc': autodoc_twilio}, - TEXTIT: {'name': u"Text It", - 'imp': imp_sub_textit, - 'imp_form': imp_sub_form_textit, - 'doc': autodoc_textit} + SMSSYNC: { + "name": "SMS Sync", + "imp": imp_sub_smssync, + "imp_form": imp_sub_form_smssync, + "doc": autodoc_smssync, + }, + TELERIVET: { + "name": "Telerivet", + "imp": imp_sub_telerivet, + "imp_form": imp_sub_form_telerivet, + "doc": autodoc_telerivet, + }, + TWILIO: { + "name": "Twilio", + "imp": imp_sub_twilio, + "imp_form": imp_sub_form_twilio, + "doc": autodoc_twilio, + }, + TEXTIT: { + "name": "Text It", + "imp": imp_sub_textit, + "imp_form": imp_sub_form_textit, + "doc": autodoc_textit, + }, } +# pylint: disable=unused-argument def unknown_service(request, username=None, id_string=None): - """ 400 view for request with unknown service name """ - r = HttpResponse(u"Unknown SMS Gateway Service", content_type='text/plain') - r.status_code = 400 - return r + """400 view for request with unknown service name""" + response = HttpResponse("Unknown SMS Gateway Service", content_type="text/plain") + response.status_code = 400 + return response @csrf_exempt def import_submission(request, username, service): - """ Proxy to the service's import_submission view """ - return PROVIDERS.get(service.lower(), {}).get( - 'imp', unknown_service)(request, username) + """Proxy to the service's import_submission view""" + return PROVIDERS.get(service.lower(), {}).get("imp", unknown_service)( + request, username + ) @csrf_exempt def import_submission_for_form(request, username, id_string, service): - """ Proxy to the service's import_submission_for_form view """ - return PROVIDERS.get(service.lower(), {}).get( - 'imp_form', unknown_service)(request, username, id_string) + """Proxy to the service's import_submission_for_form view""" + return PROVIDERS.get(service.lower(), {}).get("imp_form", unknown_service)( + request, username, id_string + ) def providers_doc(url_root, username, id_string): - return [{'id': pid, - 'name': p.get('name'), - 'doc': p.get('doc')(url_root, username, id_string)} - for pid, p in PROVIDERS.items()] + return [ + { + "id": pid, + "name": p.get("name"), + "doc": p.get("doc")(url_root, username, id_string), + } + for pid, p in PROVIDERS.items() + ] From 11ba38c11e2eb1cb57b5ffccc205e634d2872d88 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 00:35:44 +0300 Subject: [PATCH 179/234] batch: cleanup --- .../fields/instance_related_field.py | 6 +++-- .../serializers/fields/organization_field.py | 27 ++++++++++++------- .../libs/serializers/fields/project_field.py | 26 +++++++++++------- .../fields/project_related_field.py | 4 +-- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/onadata/libs/serializers/fields/instance_related_field.py b/onadata/libs/serializers/fields/instance_related_field.py index b2b40354b5..76965f908f 100644 --- a/onadata/libs/serializers/fields/instance_related_field.py +++ b/onadata/libs/serializers/fields/instance_related_field.py @@ -11,6 +11,7 @@ class InstanceRelatedField(serializers.RelatedField): """A custom field to represent the content_object generic relationship""" def get_attribute(self, instance): + """Returns instance pk.""" val = get_object_id_by_content_type(instance, Instance) if val: return val @@ -18,10 +19,11 @@ def get_attribute(self, instance): raise SkipField() def to_internal_value(self, data): + """Validates if the instance exists.""" try: return Instance.objects.get(pk=data) - except ValueError: - raise Exception("instance id should be an integer") + except ValueError as e: + raise Exception("instance id should be an integer") from e def to_representation(self, value): """Serialize instance object""" diff --git a/onadata/libs/serializers/fields/organization_field.py b/onadata/libs/serializers/fields/organization_field.py index 8c84a0314e..5e9ce1b6cf 100644 --- a/onadata/libs/serializers/fields/organization_field.py +++ b/onadata/libs/serializers/fields/organization_field.py @@ -1,24 +1,33 @@ -from builtins import str as text +# -*- coding: utf-8 -*- +""" +OrganizationField serializer field. +""" from django.utils.translation import gettext as _ + from rest_framework import serializers from onadata.apps.api.models.organization_profile import OrganizationProfile class OrganizationField(serializers.Field): - def to_representation(self, obj): - return obj.pk + def to_representation(self, value): + """Return the organization pk.""" + return value.pk def to_internal_value(self, data): + """Validate the organization exists.""" if data is not None: try: organization = OrganizationProfile.objects.get(pk=data) - except OrganizationProfile.DoesNotExist: - raise serializers.ValidationError(_( - u"Organization with id '%(value)s' does not exist." % - {"value": data} - )) + except OrganizationProfile.DoesNotExist as e: + raise serializers.ValidationError( + _( + "Organization with id '%(value)s' does not exist." + % {"value": data} + ) + ) from e except ValueError as e: - raise serializers.ValidationError(text(e)) + raise serializers.ValidationError(str(e)) from e return organization + return data diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index 190fc36931..4b2f973032 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -1,24 +1,32 @@ -from builtins import str as text +# -*- coding: utf-8 -*- +""" +ProjectField serializer field. +""" from django.utils.translation import gettext as _ + from rest_framework import serializers from onadata.apps.logger.models.project import Project class ProjectField(serializers.Field): - def to_representation(self, obj): - return obj.pk + """Project field for use with a Project object/instance.""" + + def to_representation(self, value): + """Returns the project pk.""" + return value.pk def to_internal_value(self, data): + """Validates that a project exists.""" if data is not None: try: project = Project.objects.get(pk=data) - except Project.DoesNotExist: - raise serializers.ValidationError(_( - u"Project with id '%(value)s' does not exist." % - {"value": data} - )) + except Project.DoesNotExist as e: + raise serializers.ValidationError( + _(f"Project with id '{data}' does not exist.") + ) from e except ValueError as e: - raise serializers.ValidationError(text(e)) + raise serializers.ValidationError(str(e)) from e return project + return data diff --git a/onadata/libs/serializers/fields/project_related_field.py b/onadata/libs/serializers/fields/project_related_field.py index 9451fd13f4..921a79d9d2 100644 --- a/onadata/libs/serializers/fields/project_related_field.py +++ b/onadata/libs/serializers/fields/project_related_field.py @@ -21,8 +21,8 @@ def get_attribute(self, instance): def to_internal_value(self, data): try: return Project.objects.get(pk=data) - except ValueError: - raise Exception("project id should be an integer") + except ValueError as e: + raise Exception("project id should be an integer") from e def to_representation(self, value): """Serialize project object""" From 9ff759b4f3550aebb9a0eadbba375db5a7ded046 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 00:59:52 +0300 Subject: [PATCH 180/234] batch: cleanup --- onadata/apps/sms_support/providers/smssync.py | 2 +- onadata/libs/serializers/fields/organization_field.py | 4 ++++ onadata/libs/serializers/fields/project_field.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/onadata/apps/sms_support/providers/smssync.py b/onadata/apps/sms_support/providers/smssync.py index bff0a81e1d..b146c90b04 100644 --- a/onadata/apps/sms_support/providers/smssync.py +++ b/onadata/apps/sms_support/providers/smssync.py @@ -111,7 +111,7 @@ def import_submission_for_form(request, username, id_string): ) -# pylin: disable=unused-argument +# pylint: disable=unused-argument def process_message_for_smssync(username, sms_identity, sms_text, sms_time, id_string): """Process a text instance and return in SMSSync expected format""" diff --git a/onadata/libs/serializers/fields/organization_field.py b/onadata/libs/serializers/fields/organization_field.py index 5e9ce1b6cf..50cd9932f6 100644 --- a/onadata/libs/serializers/fields/organization_field.py +++ b/onadata/libs/serializers/fields/organization_field.py @@ -10,10 +10,14 @@ class OrganizationField(serializers.Field): + """organization serializer field""" + + # pylint: disable=no-self-use def to_representation(self, value): """Return the organization pk.""" return value.pk + # pylint: disable=no-self-use def to_internal_value(self, data): """Validate the organization exists.""" if data is not None: diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index 4b2f973032..cc655479fa 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -16,6 +16,7 @@ def to_representation(self, value): """Returns the project pk.""" return value.pk + # pylint: disable=no-self-use def to_internal_value(self, data): """Validates that a project exists.""" if data is not None: From b59e5deb686130a3942b6fbf569fef723d98ca2a Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 14:40:22 +0300 Subject: [PATCH 181/234] project_cleanup.py: cleanup --- .gitignore | 2 ++ onadata/libs/serializers/fields/project_field.py | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0c8536fb7c..4947875f74 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ tags .inputrc .eggs +sonar-project.properties +.scannerwork diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index cc655479fa..e6673ae752 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -12,6 +12,7 @@ class ProjectField(serializers.Field): """Project field for use with a Project object/instance.""" + # pylint: disable=no-self-use def to_representation(self, value): """Returns the project pk.""" return value.pk From a3fbf7f6ad09118fe08fe440fb2468ec213e42ff Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 15:27:00 +0300 Subject: [PATCH 182/234] factory.py: cleanup --- onadata/apps/logger/factory.py | 199 ++++++++++++++++++--------------- 1 file changed, 109 insertions(+), 90 deletions(-) diff --git a/onadata/apps/logger/factory.py b/onadata/apps/logger/factory.py index 40504df0db..7ce6529203 100644 --- a/onadata/apps/logger/factory.py +++ b/onadata/apps/logger/factory.py @@ -1,11 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Factory utility functions. +""" # This factory is not the same as the others, and doesn't use # django-factories but it mimics their functionality... from datetime import timedelta -from pyxform import custom_values, Survey +from pyxform import Survey from pyxform.builder import create_survey_element_from_dict -from onadata.apps.logger.models import XForm, Instance +from onadata.apps.logger.models import Instance, XForm XFORM_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.000" ONE_HOUR = timedelta(0, 3600) @@ -16,22 +20,22 @@ def _load_registration_survey_object(): Loads a registration survey with all the values necessary to register a surveyor. """ - survey = Survey(name=u"registration", id_string=u"registration") - survey.add_child(create_survey_element_from_dict({ - u'type': u'text', u'name': u'name', u'label': u'Name' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'start time', - u'name': u'start' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'end time', - u'name': u'end' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'imei', - u'name': u'device_id' - })) + survey = Survey(name="registration", id_string="registration") + survey.add_child( + create_survey_element_from_dict( + {"type": "text", "name": "name", "label": "Name"} + ) + ) + survey.add_child( + create_survey_element_from_dict({"type": "start time", "name": "start"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "end time", "name": "end"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "imei", "name": "device_id"}) + ) + return survey @@ -40,96 +44,109 @@ def _load_simple_survey_object(): Returns a "watersimple" survey object, complete with questions. """ - survey = Survey(name=u"WaterSimple", id_string=u"WaterSimple") - survey.add_child(create_survey_element_from_dict({ - u'hint': {u'English': u'What is this point named?'}, - u'label': {u'English': u'Water Point Name'}, - u'type': u'text', - u'name': u'name' - })) - survey.add_child(create_survey_element_from_dict({ - u'hint': {u'English': u'How many people use this every month?'}, - u'label': {u'English': u'Monthly Usage'}, - u'name': u'users_per_month', - u'type': u'integer' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'gps', - u'name': u'geopoint', - u'label': {u'English': u'Location'} - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'imei', - u'name': u'device_id' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'start time', - u'name': u'start' - })) - survey.add_child(create_survey_element_from_dict({ - u'type': u'end time', - u'name': u'end' - })) + survey = Survey(name="WaterSimple", id_string="WaterSimple") + survey.add_child( + create_survey_element_from_dict( + { + "hint": {"English": "What is this point named?"}, + "label": {"English": "Water Point Name"}, + "type": "text", + "name": "name", + } + ) + ) + survey.add_child( + create_survey_element_from_dict( + { + "hint": {"English": "How many people use this every month?"}, + "label": {"English": "Monthly Usage"}, + "name": "users_per_month", + "type": "integer", + } + ) + ) + survey.add_child( + create_survey_element_from_dict( + {"type": "gps", "name": "geopoint", "label": {"English": "Location"}} + ) + ) + survey.add_child( + create_survey_element_from_dict({"type": "imei", "name": "device_id"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "start time", "name": "start"}) + ) + survey.add_child( + create_survey_element_from_dict({"type": "end time", "name": "end"}) + ) + return survey -class XFormManagerFactory(object): +class XFormManagerFactory: + """XForm manager factory.""" def create_registration_xform(self): """ Calls 'get_registration_xform', saves the result, and returns. """ - xf = self.get_registration_xform() - xf.save() - return xf + xform = self.get_registration_xform() + xform.save() + return xform + + # pylint: disable=no-self-use def get_registration_xform(self): """ Gets a registration xform. (currently loaded in from fixture) Returns it without saving. """ reg_xform = _load_registration_survey_object() + return XForm(xml=reg_xform.to_xml()) - def create_registration_instance(self, custom_values={}): + def create_registration_instance(self, custom_values=None): + """Create registration instance.""" + custom_values = {} if custom_values is None else custom_values i = self.get_registration_instance(custom_values) i.save() return i - def get_registration_instance(self, custom_values={}): + def get_registration_instance(self, custom_values=None): """ 1. Checks to see if the registration form has been created alread. If not, it loads it in. 2. Loads a registration instance. """ - registration_xforms = XForm.objects.filter(title=u"registration") + custom_values = {} if custom_values is None else custom_values + registration_xforms = XForm.objects.filter(title="registration") if registration_xforms.count() == 0: - xf = self.create_registration_xform() + xform = self.create_registration_xform() else: - xf = registration_xforms[0] + xform = registration_xforms[0] values = { - u'device_id': u'12345', - u'start': u'2011-01-01T09:50:06.966', - u'end': u'2011-01-01T09:53:22.965' + "device_id": "12345", + "start": "2011-01-01T09:50:06.966", + "end": "2011-01-01T09:53:22.965", } - if u'start' in custom_values: - st = custom_values[u'start'] - custom_values[u'start'] = st.strftime(XFORM_TIME_FORMAT) + if "start" in custom_values: + start = custom_values["start"] + custom_values["start"] = start.strftime(XFORM_TIME_FORMAT) # if no end_time is specified, defaults to 1 hour - values[u'end'] = (st+ONE_HOUR).strftime(XFORM_TIME_FORMAT) + values["end"] = (start + ONE_HOUR).strftime(XFORM_TIME_FORMAT) - if u'end' in custom_values: - custom_values[u'end'] = custom_values[u'end'].strftime( - XFORM_TIME_FORMAT) + if "end" in custom_values: + custom_values["end"] = custom_values["end"].strftime(XFORM_TIME_FORMAT) values.update(custom_values) reg_xform = _load_registration_survey_object() reg_instance = reg_xform.instantiate() - reg_instance._id = xf.id_string + # pylint: disable=protected-access + reg_instance._id = xform.id_string for k, v in values.items(): reg_instance.answer(name=k, value=v) @@ -139,42 +156,43 @@ def get_registration_instance(self, custom_values={}): return Instance(xml=instance_xml) def create_simple_xform(self): - xf = self.get_simple_xform() - xf.save() - return xf + """Creates and returns xform.""" + xform = self.get_simple_xform() + xform.save() + + return xform def get_simple_xform(self): + """Returns a simple xform.""" survey_object = _load_simple_survey_object() return XForm(xml=survey_object.to_xml()) - i = self.get_simple_instance(custom_values) - i.save() - return i - def get_simple_instance(self, custom_values={}): - simple_xforms = XForm.objects.filter(title=u"WaterSimple") + def get_simple_instance(self, custom_values=None): + """Returns a simple submission instance.""" + custom_values = {} if custom_values is None else custom_values + simple_xforms = XForm.objects.filter(title="WaterSimple") if simple_xforms.count() == 0: - xf = self.create_simple_xform() + xform = self.create_simple_xform() else: - xf = simple_xforms[0] + xform = simple_xforms[0] # these values can be overridden with custom values values = { - u'device_id': u'12345', - u'start': u'2011-01-01T09:50:06.966', - u'end': u'2011-01-01T09:53:22.965', - u'geopoint': u'40.783594633609184 -73.96436698913574 300.0 4.0' + "device_id": "12345", + "start": "2011-01-01T09:50:06.966", + "end": "2011-01-01T09:53:22.965", + "geopoint": "40.783594633609184 -73.96436698913574 300.0 4.0", } - if u'start' in custom_values: - st = custom_values[u'start'] - custom_values[u'start'] = st.strftime(XFORM_TIME_FORMAT) + if "start" in custom_values: + start = custom_values["start"] + custom_values["start"] = start.strftime(XFORM_TIME_FORMAT) # if no end_time is specified, defaults to 1 hour - values[u'end'] = (st+ONE_HOUR).strftime(XFORM_TIME_FORMAT) + values["end"] = (start + ONE_HOUR).strftime(XFORM_TIME_FORMAT) - if u'end' in custom_values: - custom_values[u'end'] = custom_values[u'end'].strftime( - XFORM_TIME_FORMAT) + if "end" in custom_values: + custom_values["end"] = custom_values["end"].strftime(XFORM_TIME_FORMAT) values.update(custom_values) @@ -186,7 +204,8 @@ def get_simple_instance(self, custom_values={}): # setting the id_string so that it doesn't end up # with the timestamp of the new survey object - simple_survey._id = xf.id_string + # pylint: disable=protected-access + simple_survey._id = xform.id_string instance_xml = simple_survey.to_xml() From 090feb6312511e7fc9cc6569cd33066fe0a7879e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 15:32:35 +0300 Subject: [PATCH 183/234] share_xform.py: cleanup --- onadata/libs/models/share_xform.py | 33 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/onadata/libs/models/share_xform.py b/onadata/libs/models/share_xform.py index 10e8214341..99d75b6b97 100644 --- a/onadata/libs/models/share_xform.py +++ b/onadata/libs/models/share_xform.py @@ -1,10 +1,22 @@ -from django.contrib.auth.models import User -from onadata.libs.permissions import ROLES -from onadata.libs.permissions import EditorRole, EditorMinorRole,\ - DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole +# -*- coding: utf-8 -*- +""" +ShareXForm model - facilitates sharing a form. +""" +from django.contrib.auth import get_user_model +from onadata.libs.permissions import ( + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, +) + + +class ShareXForm: + """ShareXForm class to facilitate sharing a form to a user with specified role.""" -class ShareXForm(object): def __init__(self, xform, username, role): self.xform = xform self.username = username @@ -12,14 +24,16 @@ def __init__(self, xform, username, role): @property def user(self): - return User.objects.get(username=self.username) + """Returns the user object matching ``self.username``.""" + return get_user_model().objects.get(username=self.username) + # pylint: disable=unused-argument def save(self, **kwargs): + """Assign specified role permission to a user for the given form.""" role = ROLES.get(self.role) # # check if there is xform meta perms set - meta_perms = self.xform.metadata_set\ - .filter(data_type='xform_meta_perms') + meta_perms = self.xform.metadata_set.filter(data_type="xform_meta_perms") if meta_perms: meta_perm = meta_perms[0].data_value.split("|") @@ -27,8 +41,7 @@ def save(self, **kwargs): if role in [EditorRole, EditorMinorRole]: role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role in [DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole]: role = ROLES.get(meta_perm[1]) if role and self.user and self.xform: From 009b5cc89c4b1a866fff8b5b421a8a447f207a67 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 15:46:07 +0300 Subject: [PATCH 184/234] share_team_project.py: cleanup --- onadata/libs/models/share_team_project.py | 73 +++++++++++++++-------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/onadata/libs/models/share_team_project.py b/onadata/libs/models/share_team_project.py index 675d6fa1f2..e4b8292402 100644 --- a/onadata/libs/models/share_team_project.py +++ b/onadata/libs/models/share_team_project.py @@ -1,56 +1,77 @@ -from onadata.libs.permissions import DataEntryRole, DataEntryMinorRole, \ - DataEntryOnlyRole, EditorMinorRole, EditorRole, ROLES +# -*- coding: utf-8 -*- +""" +ShareTeamProject model - facilitate sharing a project to a team. +""" +from onadata.libs.permissions import ( + ROLES, + DataEntryMinorRole, + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, +) from onadata.libs.utils.cache_tools import PROJ_PERM_CACHE, safe_delete from onadata.libs.utils.common_tags import XFORM_META_PERMS -class ShareTeamProject(object): +class ShareTeamProject: + """Share a project to a team for the given role.""" + def __init__(self, team, project, role, remove=False): self.team = team self.project = project self.role = role self.remove = remove + # pylint: disable=unused-argument def save(self, **kwargs): + """Assigns project role permissions to the team.""" + # pylint: disable=too-many-nested-blocks if self.remove: - return self.remove_team() - - role = ROLES.get(self.role) + self.remove_team() + else: + role = ROLES.get(self.role) - if role and self.team and self.project: - role.add(self.team, self.project) + if role and self.team and self.project: + role.add(self.team, self.project) - for xform in self.project.xform_set.all(): - # check if there is xform meta perms set - meta_perms = xform.metadata_set \ - .filter(data_type=XFORM_META_PERMS) - if meta_perms: - meta_perm = meta_perms[0].data_value.split("|") + for xform in self.project.xform_set.all(): + # check if there is xform meta perms set + meta_perms = xform.metadata_set.filter(data_type=XFORM_META_PERMS) + if meta_perms: + meta_perm = meta_perms[0].data_value.split("|") - if len(meta_perm) > 1: - if role in [EditorRole, EditorMinorRole]: - role = ROLES.get(meta_perm[0]) + if len(meta_perm) > 1: + if role in [EditorRole, EditorMinorRole]: + role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: - role = ROLES.get(meta_perm[1]) - role.add(self.team, xform) + elif role in [ + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, + ]: + role = ROLES.get(meta_perm[1]) + role.add(self.team, xform) - for dataview in self.project.dataview_set.all(): - if dataview.matches_parent: - role.add(self.user, dataview.xform) + for dataview in self.project.dataview_set.all(): + if dataview.matches_parent: + role.add(self.team, dataview.xform) - # clear cache - safe_delete('{}{}'.format(PROJ_PERM_CACHE, self.project.pk)) + # clear cache + safe_delete(f"{PROJ_PERM_CACHE}{self.project.pk}") def remove_team(self): + """Removes team permissions from a project.""" role = ROLES.get(self.role) if role and self.team and self.project: + # pylint: disable=protected-access role._remove_obj_permissions(self.team, self.project) for xform in self.project.xform_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(self.team, xform) for dataview in self.project.dataview_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(self.team, dataview.xform) From e81c196c1b47821d5c3293a4b46e107851ef8c8e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 15:58:09 +0300 Subject: [PATCH 185/234] import_forms.py: cleanup --- .../logger/management/commands/import_forms.py | 15 +++++++++++---- .../viewer/management/commands/import_forms.py | 14 ++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/onadata/apps/logger/management/commands/import_forms.py b/onadata/apps/logger/management/commands/import_forms.py index 6fd8d72d62..f33a592042 100644 --- a/onadata/apps/logger/management/commands/import_forms.py +++ b/onadata/apps/logger/management/commands/import_forms.py @@ -1,5 +1,9 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +""" +import_forms - loads XForms from a given path. +""" from __future__ import absolute_import import glob @@ -12,11 +16,14 @@ class Command(BaseCommand): + """Import a folder of XForms for ODK.""" + help = gettext_lazy("Import a folder of XForms for ODK.") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): + """Import a folder of XForms for ODK.""" path = args[0] for form in glob.glob(os.path.join(path, "*")): - f = open(form) - XForm.objects.get_or_create(xml=f.read(), downloadable=False) - f.close() + with open(form, encoding="utf-8") as f: + XForm.objects.get_or_create(xml=f.read(), downloadable=False) diff --git a/onadata/apps/viewer/management/commands/import_forms.py b/onadata/apps/viewer/management/commands/import_forms.py index 9be917d775..0da97d6171 100644 --- a/onadata/apps/viewer/management/commands/import_forms.py +++ b/onadata/apps/viewer/management/commands/import_forms.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 +# vim: ai ts=4 sts=4 et sw=4 +""" +import_forms - loads XForms from a given path. +""" from __future__ import absolute_import import glob @@ -12,11 +15,14 @@ class Command(BaseCommand): + """Import a folder of XForms for ODK.""" + help = gettext_lazy("Import a folder of XForms for ODK.") + # pylint: disable=unused-argument def handle(self, *args, **kwargs): + """Import a folder of XForms for ODK.""" path = args[0] for form in glob.glob(os.path.join(path, "*")): - f = open(form) - XForm.objects.get_or_create(xml=f.read(), active=False) - f.close() + with open(form, encoding="utf-8") as f: + XForm.objects.get_or_create(xml=f.read(), active=False) From 0dff8cbffbff2d163ca290601859b0e88637b5e5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 16:30:49 +0300 Subject: [PATCH 186/234] reassign_permission.py: cleanup --- .../commands/reassign_permission.py | 75 +++++++++++++------ .../management/commands/import_forms.py | 1 + 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/onadata/apps/api/management/commands/reassign_permission.py b/onadata/apps/api/management/commands/reassign_permission.py index bd9dd10697..38ee347f10 100644 --- a/onadata/apps/api/management/commands/reassign_permission.py +++ b/onadata/apps/api/management/commands/reassign_permission.py @@ -1,30 +1,50 @@ +# -*- coding: utf-8 -*- +""" +reassign_permission - reassign permission to the model when permissions are changed. +""" from guardian.shortcuts import get_perms from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from django.conf import settings from onadata.apps.api.models import Team -from onadata.libs.permissions import ReadOnlyRole, DataEntryRole,\ - EditorRole, ManagerRole, OwnerRole, ReadOnlyRoleNoDownload,\ - DataEntryOnlyRole, DataEntryMinorRole, EditorMinorRole +from onadata.libs.permissions import ( + ReadOnlyRole, + DataEntryRole, + EditorRole, + ManagerRole, + OwnerRole, + ReadOnlyRoleNoDownload, + DataEntryOnlyRole, + DataEntryMinorRole, + EditorMinorRole, +) from onadata.libs.utils.model_tools import queryset_iterator +# pylint: disable=invalid-name +User = get_user_model() + + class Command(BaseCommand): - args = '' - help = _(u"Reassign permission to the model when permissions are changed") + """Reassign permission to the model when permissions are changed""" + args = "" + help = _("Reassign permission to the model when permissions are changed") + + # pylint: disable=unused-argument def handle(self, *args, **options): - self.stdout.write("Re-assigining started", ending='\n') + """Reassign permission to the model when permissions are changed""" + self.stdout.write("Re-assigining started", ending="\n") if not args: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") if len(args) < 3: - raise CommandError('Param not set. ') + raise CommandError("Param not set. ") app = args[0] model = args[1] @@ -47,7 +67,7 @@ def handle(self, *args, **options): for team in queryset_iterator(teams): self.reassign_perms(team, app, model, new_perms) - self.stdout.write("Re-assigining finished", ending='\n') + self.stdout.write("Re-assigining finished", ending="\n") def reassign_perms(self, user, app, model, new_perm): """ @@ -65,37 +85,42 @@ def reassign_perms(self, user, app, model, new_perm): if isinstance(user, Team): if model == "project": objects = user.projectgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: objects = user.xformgroupobjectpermission_set.filter( - group_id=user.pk).distinct('content_object_id') + group_id=user.pk + ).distinct("content_object_id") else: - if model == 'project': + if model == "project": objects = user.projectuserobjectpermission_set.all() else: objects = user.xformuserobjectpermission_set.all() for perm_obj in objects: obj = perm_obj.content_object - ROLES = [ReadOnlyRoleNoDownload, - ReadOnlyRole, - DataEntryOnlyRole, - DataEntryMinorRole, - DataEntryRole, - EditorMinorRole, - EditorRole, - ManagerRole, - OwnerRole] + roles = [ + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, + ] # For each role reassign the perms - for role_class in reversed(ROLES): + for role_class in reversed(roles): if self.check_role(role_class, user, obj, new_perm): # If true role_class.add(user, obj) break - def check_role(self, role_class, user, obj, new_perm=[]): + # pylint: disable=no-self-use + def check_role(self, role_class, user, obj, new_perm=None): """ Test if the user has the role for the object provided :param role_class: @@ -104,6 +129,7 @@ def check_role(self, role_class, user, obj, new_perm=[]): :param new_perm: :return: """ + new_perm = [] if new_perm is None else new_perm # remove the new permission because the old model doesnt have it perm_list = role_class.class_to_permissions[type(obj)] old_perm_set = set(perm_list) @@ -116,3 +142,4 @@ def check_role(self, role_class, user, obj, new_perm=[]): return set(get_perms(user, obj)) == diff_set return user.has_perms(list(diff_set), obj) + return False diff --git a/onadata/apps/viewer/management/commands/import_forms.py b/onadata/apps/viewer/management/commands/import_forms.py index 0da97d6171..e415f6e1c7 100644 --- a/onadata/apps/viewer/management/commands/import_forms.py +++ b/onadata/apps/viewer/management/commands/import_forms.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 """ import_forms - loads XForms from a given path. From d269018419d5942debbb3f2a731560cf6b41069c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 16:35:06 +0300 Subject: [PATCH 187/234] share_project.py: cleanup --- onadata/libs/models/share_project.py | 50 +++++++++++++++++++++------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/onadata/libs/models/share_project.py b/onadata/libs/models/share_project.py index 7cea9ded74..5ea246cf57 100644 --- a/onadata/libs/models/share_project.py +++ b/onadata/libs/models/share_project.py @@ -1,25 +1,45 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +ShareProject model - facilitate sharing of a project to a user. +""" +from django.contrib.auth import get_user_model from django.db import transaction from onadata.libs.permissions import ROLES -from onadata.libs.permissions import EditorRole, EditorMinorRole,\ - DataEntryRole, DataEntryMinorRole, DataEntryOnlyRole +from onadata.libs.permissions import ( + EditorRole, + EditorMinorRole, + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, +) from onadata.libs.utils.cache_tools import ( - PROJ_PERM_CACHE, PROJ_OWNER_CACHE, safe_delete) + PROJ_PERM_CACHE, + PROJ_OWNER_CACHE, + safe_delete, +) + +# pylint: disable=invalid-name +User = get_user_model() def remove_xform_permissions(project, user, role): + """Remove user permissions to all forms for the given ``project``.""" # remove role from project forms as well for xform in project.xform_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(user, xform) def remove_dataview_permissions(project, user, role): + """Remove user permissions to all dataviews for the given ``project``.""" for dataview in project.dataview_set.all(): + # pylint: disable=protected-access role._remove_obj_permissions(user, dataview.xform) -class ShareProject(object): +class ShareProject: + """Share project with a user.""" def __init__(self, project, username, role, remove=False): self.project = project @@ -29,11 +49,14 @@ def __init__(self, project, username, role, remove=False): @property def user(self): + """Return the user object for the given ``self.username``.""" return User.objects.get(username=self.username) + # pylint: disable=unused-argument @transaction.atomic() def save(self, **kwargs): - + """Assigns role permissions to a project for the user.""" + # pylint: disable=too-many-nested-blocks if self.remove: self.__remove_user() else: @@ -45,8 +68,7 @@ def save(self, **kwargs): # apply same role to forms under the project for xform in self.project.xform_set.all(): # check if there is xform meta perms set - meta_perms = xform.metadata_set \ - .filter(data_type='xform_meta_perms') + meta_perms = xform.metadata_set.filter(data_type="xform_meta_perms") if meta_perms: meta_perm = meta_perms[0].data_value.split("|") @@ -54,8 +76,11 @@ def save(self, **kwargs): if role in [EditorRole, EditorMinorRole]: role = ROLES.get(meta_perm[0]) - elif role in [DataEntryRole, DataEntryMinorRole, - DataEntryOnlyRole]: + elif role in [ + DataEntryRole, + DataEntryMinorRole, + DataEntryOnlyRole, + ]: role = ROLES.get(meta_perm[1]) role.add(self.user, xform) @@ -64,8 +89,8 @@ def save(self, **kwargs): role.add(self.user, dataview.xform) # clear cache - safe_delete('{}{}'.format(PROJ_OWNER_CACHE, self.project.pk)) - safe_delete('{}{}'.format(PROJ_PERM_CACHE, self.project.pk)) + safe_delete(f"{PROJ_OWNER_CACHE}{self.project.pk}") + safe_delete(f"{PROJ_PERM_CACHE}{self.project.pk}") @transaction.atomic() def __remove_user(self): @@ -74,4 +99,5 @@ def __remove_user(self): if role and self.user and self.project: remove_xform_permissions(self.project, self.user, role) remove_dataview_permissions(self.project, self.user, role) + # pylint: disable=protected-access role._remove_obj_permissions(self.user, self.project) From 0cf040f6d6048d55c113e0ba91fcc23499de6d59 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 16:42:26 +0300 Subject: [PATCH 188/234] merged_xform.py: cleanup --- onadata/apps/logger/models/merged_xform.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/onadata/apps/logger/models/merged_xform.py b/onadata/apps/logger/models/merged_xform.py index 26e6c05145..188a37bc9b 100644 --- a/onadata/apps/logger/models/merged_xform.py +++ b/onadata/apps/logger/models/merged_xform.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +MergedXForm model - stores info on forms to merge. +""" from django.db import models from django.db.models.signals import post_save @@ -10,20 +14,24 @@ class MergedXForm(XForm): Merged XForms """ - xforms = models.ManyToManyField( - 'logger.XForm', related_name='mergedxform_ptr') + xforms = models.ManyToManyField("logger.XForm", related_name="mergedxform_ptr") class Meta: - app_label = 'logger' + app_label = "logger" def save(self, *args, **kwargs): set_uuid(self) - return super(MergedXForm, self).save(*args, **kwargs) + return super().save(*args, **kwargs) +# pylint: disable=unused-argument def set_object_permissions(sender, instance=None, created=False, **kwargs): + """Set object permissions when a MergedXForm has been created.""" + if created: + # pylint: disable=import-outside-toplevel from onadata.libs.permissions import OwnerRole + OwnerRole.add(instance.user, instance) OwnerRole.add(instance.user, instance.xform_ptr) @@ -32,6 +40,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): OwnerRole.add(instance.created_by, instance.xform_ptr) from onadata.libs.utils.project_utils import set_project_perms_to_xform + set_project_perms_to_xform(instance, instance.project) set_project_perms_to_xform(instance.xform_ptr, instance.project) @@ -39,4 +48,5 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): post_save.connect( set_object_permissions, sender=MergedXForm, - dispatch_uid='set_project_perms_to_merged_xform') + dispatch_uid="set_project_perms_to_merged_xform", +) From cbd3390187d09467cad70598967a5d252aa99ad0 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 17:03:33 +0300 Subject: [PATCH 189/234] clone_xform.py: cleanup --- onadata/libs/models/clone_xform.py | 35 +++++++++++-------- .../serializers/clone_xform_serializer.py | 33 ++++++++++++----- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/onadata/libs/models/clone_xform.py b/onadata/libs/models/clone_xform.py index 467c6b8539..d618837eb3 100644 --- a/onadata/libs/models/clone_xform.py +++ b/onadata/libs/models/clone_xform.py @@ -1,33 +1,40 @@ -from django.contrib.auth.models import User -from onadata.apps.viewer.models.data_dictionary import \ - DataDictionary, upload_to +# -*- coding: utf-8 -*- +""" +CloneXForm class model. +""" +from django.contrib.auth import get_user_model from django.core.files.storage import default_storage + from onadata.apps.logger.models.xform import XForm +from onadata.apps.viewer.models.data_dictionary import DataDictionary, upload_to from onadata.libs.utils.user_auth import get_user_default_project -class CloneXForm(object): +class CloneXForm: + """The class takes an existing form's XLSForm and publishes it as a new form.""" + def __init__(self, xform, username, project=None): self.xform = xform self.username = username self.project = project + self.cloned_form = None @property def user(self): - return User.objects.get(username=self.username) + """Returns a User object for the given ``self.username``.""" + return get_user_model().objects.get(username=self.username) def save(self, **kwargs): - user = User.objects.get(username=self.username) + """Publishes an exiting form's XLSForm as a new form.""" + user = self.user project = self.project or get_user_default_project(user) - xls_file_path = upload_to(None, '%s%s.xlsx' % ( - self.xform.id_string, - XForm.CLONED_SUFFIX), - self.username) + xls_file_path = upload_to( + None, + f"{self.xform.id_string}{XForm.CLONED_SUFFIX}.xlsx", + self.username, + ) xls_data = default_storage.open(self.xform.xls.name) xls_file = default_storage.save(xls_file_path, xls_data) self.cloned_form = DataDictionary.objects.create( - user=user, - created_by=user, - xls=xls_file, - project=project + user=user, created_by=user, xls=xls_file, project=project ) diff --git a/onadata/libs/serializers/clone_xform_serializer.py b/onadata/libs/serializers/clone_xform_serializer.py index 763b1a5da8..0392a41342 100644 --- a/onadata/libs/serializers/clone_xform_serializer.py +++ b/onadata/libs/serializers/clone_xform_serializer.py @@ -1,38 +1,53 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +Clone an XForm serializer. +""" +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.clone_xform import CloneXForm -from onadata.libs.serializers.fields.xform_field import XFormField from onadata.libs.serializers.fields.project_field import ProjectField +from onadata.libs.serializers.fields.xform_field import XFormField class CloneXFormSerializer(serializers.Serializer): + """Clone an xform serializer class""" + xform = XFormField() username = serializers.CharField(max_length=255) project = ProjectField(required=False) + # pylint: disable=no-self-use def create(self, validated_data): + """Uses the CloneXForm class to clone/copy an XForm. + + Returns the CloneXForm instance.""" instance = CloneXForm(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def update(self, instance, validated_data): - instance.xform = validated_data.get('xform', instance.xform) - instance.username = validated_data.get('username', instance.username) - instance.project = validated_data.get('project', instance.project) + instance.xform = validated_data.get("xform", instance.xform) + instance.username = validated_data.get("username", instance.username) + instance.project = validated_data.get("project", instance.project) instance.save() return instance + # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: User.objects.get(username=value) - except User.DoesNotExist: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + except User.DoesNotExist as e: + raise serializers.ValidationError( + _(f"User '{value}' does not exist.") + ) from e return value From 941d291ca0afa01f588ec74783d767ab4c3c5fe2 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 17:18:46 +0300 Subject: [PATCH 190/234] share_xform_serializer.py: cleanup --- .../serializers/share_xform_serializer.py | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/onadata/libs/serializers/share_xform_serializer.py b/onadata/libs/serializers/share_xform_serializer.py index 0cc0592e9b..c1c0f9e9e9 100644 --- a/onadata/libs/serializers/share_xform_serializer.py +++ b/onadata/libs/serializers/share_xform_serializer.py @@ -1,47 +1,60 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +Share XForm serializer. +""" +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.share_xform import ShareXForm from onadata.libs.permissions import ROLES from onadata.libs.serializers.fields.xform_field import XFormField class ShareXFormSerializer(serializers.Serializer): + """Share xform to a user.""" + xform = XFormField() username = serializers.CharField(max_length=255) role = serializers.CharField(max_length=50) + # pylint: disable=unused-argument,no-self-use def update(self, instance, validated_data): - instance.xform = validated_data.get('xform', instance.xform) - instance.username = validated_data.get('username', instance.username) - instance.role = validated_data.get('role', instance.role) + """Make changes to form share to a user.""" + instance.xform = validated_data.get("xform", instance.xform) + instance.username = validated_data.get("username", instance.username) + instance.role = validated_data.get("role", instance.role) instance.save() return instance + # pylint: disable=unused-argument,no-self-use def create(self, validated_data): + """Assign role permission for a form to a user.""" instance = ShareXForm(**validated_data) instance.save() return instance + # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" + # pylint: disable=invalid-name + User = get_user_model() # noqa N806 try: User.objects.get(username=value) - except User.DoesNotExist: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + except User.DoesNotExist as e: + raise serializers.ValidationError( + _(f"User '{value}' does not exist.") + ) from e return value + # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value From 990e8a44c594c530564469e35ca982030e8fb849 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 17:34:59 +0300 Subject: [PATCH 191/234] Duplicate command. --- .../management/commands/import_forms.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 onadata/apps/viewer/management/commands/import_forms.py diff --git a/onadata/apps/viewer/management/commands/import_forms.py b/onadata/apps/viewer/management/commands/import_forms.py deleted file mode 100644 index e415f6e1c7..0000000000 --- a/onadata/apps/viewer/management/commands/import_forms.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# vim: ai ts=4 sts=4 et sw=4 -""" -import_forms - loads XForms from a given path. -""" -from __future__ import absolute_import - -import glob -import os - -from django.core.management.base import BaseCommand -from django.utils.translation import gettext_lazy - -from onadata.apps.logger.models import XForm - - -class Command(BaseCommand): - """Import a folder of XForms for ODK.""" - - help = gettext_lazy("Import a folder of XForms for ODK.") - - # pylint: disable=unused-argument - def handle(self, *args, **kwargs): - """Import a folder of XForms for ODK.""" - path = args[0] - for form in glob.glob(os.path.join(path, "*")): - with open(form, encoding="utf-8") as f: - XForm.objects.get_or_create(xml=f.read(), active=False) From 1f40551a249cc888c0e5e7e00e01f4760c92189d Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 17:51:01 +0300 Subject: [PATCH 192/234] __init__.py cleanup --- .flake8 | 1 + onadata/apps/api/models/__init__.py | 13 ++++++++++--- onadata/apps/logger/models/__init__.py | 24 ++++++++++++++---------- onadata/apps/main/models/__init__.py | 4 ++++ onadata/apps/viewer/models/__init__.py | 4 ++++ 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.flake8 b/.flake8 index ebaca2e124..44033a966d 100644 --- a/.flake8 +++ b/.flake8 @@ -3,3 +3,4 @@ max-line-length = 88 ... select = C,E,F,W,B,B950 extend-ignore = E203, E501 +per-file-ignores = __init__.py:F401 diff --git a/onadata/apps/api/models/__init__.py b/onadata/apps/api/models/__init__.py index 6b89a4b54b..2e109ccbe8 100644 --- a/onadata/apps/api/models/__init__.py +++ b/onadata/apps/api/models/__init__.py @@ -1,3 +1,10 @@ -from onadata.apps.api.models.organization_profile import OrganizationProfile # noqa -from onadata.apps.api.models.team import Team # noqa -from onadata.apps.api.models.temp_token import TempToken # noqa +# -*- coding: utf-8 -*- +""" +API models. +""" + +from onadata.apps.api.models.organization_profile import ( + OrganizationProfile, +) # noqa F401 +from onadata.apps.api.models.team import Team # noqa F401 +from onadata.apps.api.models.temp_token import TempToken # noqa F401 diff --git a/onadata/apps/logger/models/__init__.py b/onadata/apps/logger/models/__init__.py index 6dab34889d..7d85c6d6be 100644 --- a/onadata/apps/logger/models/__init__.py +++ b/onadata/apps/logger/models/__init__.py @@ -1,14 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Logger models. +""" from onadata.apps.logger.models.attachment import Attachment # noqa from onadata.apps.logger.models.data_view import DataView # noqa from onadata.apps.logger.models.instance import Instance # noqa from onadata.apps.logger.models.merged_xform import MergedXForm # noqa -from onadata.apps.logger.models.note import Note # noqa -from onadata.apps.logger.models.open_data import OpenData # noqa -from onadata.apps.logger.models.osmdata import OsmData # noqa -from onadata.apps.logger.models.project import Project # noqa -from onadata.apps.logger.models.survey_type import SurveyType # noqa -from onadata.apps.logger.models.widget import Widget # noqa -from onadata.apps.logger.models.xform import XForm # noqa -from onadata.apps.logger.models.submission_review import SubmissionReview # noqa -from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa -from onadata.apps.logger.models.xform_version import XFormVersion # noqa +from onadata.apps.logger.models.note import Note # noqa +from onadata.apps.logger.models.open_data import OpenData # noqa +from onadata.apps.logger.models.osmdata import OsmData # noqa +from onadata.apps.logger.models.project import Project # noqa +from onadata.apps.logger.models.survey_type import SurveyType # noqa +from onadata.apps.logger.models.widget import Widget # noqa +from onadata.apps.logger.models.xform import XForm # noqa +from onadata.apps.logger.models.submission_review import SubmissionReview # noqa +from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa +from onadata.apps.logger.models.xform_version import XFormVersion # noqa diff --git a/onadata/apps/main/models/__init__.py b/onadata/apps/main/models/__init__.py index 8a4d74aff2..f342a92dc5 100644 --- a/onadata/apps/main/models/__init__.py +++ b/onadata/apps/main/models/__init__.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Main models. +""" from __future__ import absolute_import from onadata.apps.main.models.user_profile import UserProfile # noqa diff --git a/onadata/apps/viewer/models/__init__.py b/onadata/apps/viewer/models/__init__.py index 9d4fc352e4..fb1341abfd 100644 --- a/onadata/apps/viewer/models/__init__.py +++ b/onadata/apps/viewer/models/__init__.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Viewer models. +""" from onadata.apps.viewer.models.parsed_instance import ParsedInstance # noqa from onadata.apps.viewer.models.data_dictionary import DataDictionary # noqa from onadata.apps.viewer.models.export import Export # noqa From c87a51bfab0d7287d8c037553c36bb9cbc79e1a9 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sun, 8 May 2022 18:32:58 +0300 Subject: [PATCH 193/234] __init__.py cleanup --- onadata/apps/api/models/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/onadata/apps/api/models/__init__.py b/onadata/apps/api/models/__init__.py index 2e109ccbe8..2366b4ba36 100644 --- a/onadata/apps/api/models/__init__.py +++ b/onadata/apps/api/models/__init__.py @@ -2,9 +2,8 @@ """ API models. """ +# flake8: noqa -from onadata.apps.api.models.organization_profile import ( - OrganizationProfile, -) # noqa F401 -from onadata.apps.api.models.team import Team # noqa F401 -from onadata.apps.api.models.temp_token import TempToken # noqa F401 +from onadata.apps.api.models.organization_profile import OrganizationProfile # noqa +from onadata.apps.api.models.team import Team # noqa +from onadata.apps.api.models.temp_token import TempToken # noqa From 3ffa28162540f86d5d2e3892a309cd73b8bee58c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 10 May 2022 17:58:35 +0300 Subject: [PATCH 194/234] signal.py: Check ASYNC_POST_SUBMISSION_PROCESSING_ENABLED --- onadata/apps/restservice/signals.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/onadata/apps/restservice/signals.py b/onadata/apps/restservice/signals.py index c15d3fb67d..7782abbb17 100644 --- a/onadata/apps/restservice/signals.py +++ b/onadata/apps/restservice/signals.py @@ -7,19 +7,23 @@ from onadata.apps.restservice.tasks import call_service_async -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) # pylint: disable=C0103 -trigger_webhook = django.dispatch.Signal(providing_args=['instance']) +trigger_webhook = django.dispatch.Signal(providing_args=["instance"]) def call_webhooks(sender, **kwargs): # pylint: disable=W0613 """ Call webhooks signal. """ - instance_id = kwargs['instance'].pk - call_service_async.apply_async(args=[instance_id], countdown=1) + instance_id = kwargs["instance"].pk + if ASYNC_POST_SUBMISSION_PROCESSING_ENABLED: + call_service_async.apply_async(args=[instance_id], countdown=1) + else: + call_service_async(instance_id) -trigger_webhook.connect(call_webhooks, dispatch_uid='call_webhooks') +trigger_webhook.connect(call_webhooks, dispatch_uid="call_webhooks") From b0d91865022389e43e239d2eefdeda863b65a9d6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 10 May 2022 17:59:22 +0300 Subject: [PATCH 195/234] csv_import.py: Check if xform.json is a dict --- onadata/libs/utils/csv_import.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 484c11a925..fa605aa999 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -101,11 +101,12 @@ def dict2xmlsubmission(submission_dict, xform, instance_id, submission_date): :rtype: string """ + xform_dict = xform.json if isinstance(xform.json, dict) else json.loads(xform.json) return ( '' '<{0} id="{1}" instanceID="uuid:{2}" submissionDate="{3}">{4}' "".format( - json.loads(xform.json).get("name", xform.id_string), + xform_dict.get("name", xform.id_string), xform.id_string, instance_id, submission_date, @@ -323,7 +324,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): csv_file.seek(0) csv_reader = ucsv.DictReader(csv_file, encoding="utf-8-sig") - xform_json = json.loads(xform.json) + xform_json = xform.json if isinstance(xform.json, dict) else json.loads(xform.json) select_multiples = [ qstn.name for qstn in xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE) ] From 5ed6b01d936eb5c6c050ec87684f4915fa73f5d6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 10 May 2022 22:59:39 +0300 Subject: [PATCH 196/234] restservice: cleanup --- .../apps/restservice/RestServiceInterface.py | 11 ++- onadata/apps/restservice/__init__.py | 16 +++- onadata/apps/restservice/forms.py | 16 +++- .../management/commands/textit_v1_to_v2.py | 53 ++++++----- onadata/apps/restservice/models.py | 10 +- onadata/apps/restservice/services/f2dhis2.py | 27 ++++-- .../apps/restservice/services/generic_json.py | 21 ++-- .../apps/restservice/services/generic_xml.py | 20 +++- onadata/apps/restservice/services/textit.py | 19 +++- onadata/apps/restservice/signals.py | 6 +- onadata/apps/restservice/tasks.py | 7 +- .../restservice/tests/test_restservice.py | 35 +++++-- .../viewsets/test_restservicesviewset.py | 43 ++++----- onadata/apps/restservice/utils.py | 24 +++-- onadata/apps/restservice/views.py | 95 +++++++++++-------- .../viewsets/restservices_viewset.py | 48 ++++++---- 16 files changed, 285 insertions(+), 166 deletions(-) diff --git a/onadata/apps/restservice/RestServiceInterface.py b/onadata/apps/restservice/RestServiceInterface.py index f7d995d5fd..dca06ec26e 100644 --- a/onadata/apps/restservice/RestServiceInterface.py +++ b/onadata/apps/restservice/RestServiceInterface.py @@ -1,3 +1,12 @@ -class RestServiceInterface(object): +# -*- coding: utf-8 -*- +""" +Base class. +""" + + +class RestServiceInterface: + """RestServiceInterface base class.""" + def send(self, url, data=None): + """The class method to implement when sending data.""" raise NotImplementedError diff --git a/onadata/apps/restservice/__init__.py b/onadata/apps/restservice/__init__.py index 267a9cbd4a..59ca345a6e 100644 --- a/onadata/apps/restservice/__init__.py +++ b/onadata/apps/restservice/__init__.py @@ -1,4 +1,12 @@ -SERVICE_CHOICES = ((u'f2dhis2', u'f2dhis2'), (u'generic_json', u'JSON POST'), - (u'generic_xml', u'XML POST'), (u'bamboo', u'bamboo'), - (u'textit', u'TextIt POST'), - (u'google_sheets', u'Google Sheet')) +# -*- coding: utf-8 -*- +""" +restservice module. +""" +SERVICE_CHOICES = ( + ("f2dhis2", "f2dhis2"), + ("generic_json", "JSON POST"), + ("generic_xml", "XML POST"), + ("bamboo", "bamboo"), + ("textit", "TextIt POST"), + ("google_sheets", "Google Sheet"), +) diff --git a/onadata/apps/restservice/forms.py b/onadata/apps/restservice/forms.py index 2f6d052fa8..19c088347f 100644 --- a/onadata/apps/restservice/forms.py +++ b/onadata/apps/restservice/forms.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +restservice forms. +""" from django import forms from django.utils.translation import gettext_lazy @@ -5,7 +9,11 @@ class RestServiceForm(forms.Form): - service_name = \ - forms.CharField(max_length=50, label=gettext_lazy(u"Service Name"), - widget=forms.Select(choices=SERVICE_CHOICES)) - service_url = forms.URLField(label=gettext_lazy(u"Service URL")) + """RestService form class.""" + + service_name = forms.CharField( + max_length=50, + label=gettext_lazy("Service Name"), + widget=forms.Select(choices=SERVICE_CHOICES), + ) + service_url = forms.URLField(label=gettext_lazy("Service URL")) diff --git a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py index 4248598302..0fddffaac9 100644 --- a/onadata/apps/restservice/management/commands/textit_v1_to_v2.py +++ b/onadata/apps/restservice/management/commands/textit_v1_to_v2.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +textit_v1_to_v2 - converts RapidPro/textit urls from v1 to v2 urls. +""" import re @@ -10,33 +14,38 @@ class Command(BaseCommand): + """Migrate TextIt/RapidPro v1 URLS to v2 URLS.""" + help = _("Migrate TextIt/RapidPro v1 URLS to v2 URLS.") def add_arguments(self, parser): parser.add_argument( - '--apply', default=False, help=_("Apply changes to database.")) + "--apply", default=False, help=_("Apply changes to database.") + ) + # pylint: disable=unused-argument def handle(self, *args, **options): + """Migrate TextIt/RapidPro v1 URLS to v2 URLS.""" services = RestService.objects.filter(name=TEXTIT) - force = options.get('apply') - if force and force.lower() != 'true': + force = options.get("apply") + if force and force.lower() != "true": self.stderr.write("--apply expects 'true' as a parameter value.") - - return - - v1 = re.compile(r'\/v1/runs') - v2 = '/v2/flow_starts' - - for service in services: - if v1.findall(service.service_url): - original = service.service_url - new_uri = re.sub(v1, v2, service.service_url) - params = {'v1_url': original, 'v2_url': new_uri} - if force.lower() == 'true': - service.service_url = new_uri - service.save() - self.stdout.write( - _("Changed %(v1_url)s to %(v2_url)s" % params)) - else: - self.stdout.write( - _("Will change %(v1_url)s to %(v2_url)s" % params)) + else: + version_1 = re.compile(r"\/v1/runs") + version_2 = "/v2/flow_starts" + + for service in services: + if version_1.findall(service.service_url): + original = service.service_url + new_uri = re.sub(version_1, version_2, service.service_url) + params = {"v1_url": original, "v2_url": new_uri} + if force.lower() == "true": + service.service_url = new_uri + service.save() + self.stdout.write( + _("Changed %(v1_url)s to %(v2_url)s" % params) + ) + else: + self.stdout.write( + _("Will change %(v1_url)s to %(v2_url)s" % params) + ) diff --git a/onadata/apps/restservice/models.py b/onadata/apps/restservice/models.py index 73c27ab23d..bcafa85f61 100644 --- a/onadata/apps/restservice/models.py +++ b/onadata/apps/restservice/models.py @@ -37,7 +37,7 @@ class Meta: ) def __str__(self): - return "%s:%s - %s" % (self.xform, self.long_name, self.service_url) + return f"{self.xform}:{self.long_name} - {self.service_url}" def get_service_definition(self): """ @@ -45,7 +45,7 @@ def get_service_definition(self): """ services_to_modules = getattr(settings, "REST_SERVICES_TO_MODULES", {}) module_name = services_to_modules.get( - self.name, "onadata.apps.restservice.services.%s" % self.name + self.name, f"onadata.apps.restservice.services.{self.name}" ) module = importlib.import_module(module_name) @@ -61,7 +61,7 @@ def long_name(self): return service_definition.verbose_name -def delete_metadata(sender, instance, **kwargs): # pylint: disable=W0613 +def delete_metadata(sender, instance, **kwargs): # pylint: disable=unused-argument """ Delete related metadata on deletion of the RestService. """ @@ -74,7 +74,7 @@ def delete_metadata(sender, instance, **kwargs): # pylint: disable=W0613 post_delete.connect(delete_metadata, sender=RestService, dispatch_uid="delete_metadata") -# pylint: disable=W0613 +# pylint: disable=unused-argument def propagate_merged_datasets(sender, instance, **kwargs): """ Propagate the service to the individual forms of a merged dataset. @@ -94,7 +94,7 @@ def propagate_merged_datasets(sender, instance, **kwargs): ) -# pylint: disable=W0613 +# pylint: disable=unused-argument def delete_merged_datasets_service(sender, instance, **kwargs): """ Delete the service to the individual forms of a merged dataset. diff --git a/onadata/apps/restservice/services/f2dhis2.py b/onadata/apps/restservice/services/f2dhis2.py index eb62f473d2..c80171c04a 100644 --- a/onadata/apps/restservice/services/f2dhis2.py +++ b/onadata/apps/restservice/services/f2dhis2.py @@ -1,16 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Formhub/Ona Data to DHIS2 service - push submissions to DHIS2 instance. +""" import requests from onadata.apps.restservice.RestServiceInterface import RestServiceInterface class ServiceDefinition(RestServiceInterface): - id = u'f2dhis2' - verbose_name = u'Formhub to DHIS2' + """Post submission to DHIS2 instance.""" - def send(self, url, submission_instance): - info = { - "id_string": submission_instance.xform.id_string, - "uuid": submission_instance.uuid - } - valid_url = url % info - requests.get(valid_url) + # pylint: disable=invalid-name + id = "f2dhis2" + verbose_name = "Formhub to DHIS2" + + def send(self, url, data=None): + """Post submission to DHIS2 instance.""" + if data: + info = { + "id_string": data.xform.id_string, + "uuid": data.uuid, + } + valid_url = url % info + requests.get(valid_url) diff --git a/onadata/apps/restservice/services/generic_json.py b/onadata/apps/restservice/services/generic_json.py index b1fa30c428..24c97e693d 100644 --- a/onadata/apps/restservice/services/generic_json.py +++ b/onadata/apps/restservice/services/generic_json.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Post submisison JSON data to an external service that accepts a JSON post. +""" import json import requests @@ -6,10 +10,15 @@ class ServiceDefinition(RestServiceInterface): - id = u'json' - verbose_name = u'JSON POST' + """Post submisison JSON data to an external service that accepts a JSON post.""" - def send(self, url, submission_instance): - post_data = json.dumps(submission_instance.json) - headers = {"Content-Type": "application/json"} - requests.post(url, headers=headers, data=post_data) + # pylint: disable=invalid-name + id = "json" + verbose_name = "JSON POST" + + def send(self, url, data=None): + """Post submisison JSON data to an external service that accepts a JSON post.""" + if data: + post_data = json.dumps(data.json) + headers = {"Content-Type": "application/json"} + requests.post(url, headers=headers, data=post_data) diff --git a/onadata/apps/restservice/services/generic_xml.py b/onadata/apps/restservice/services/generic_xml.py index 9a9fb212df..2ac3104cd4 100644 --- a/onadata/apps/restservice/services/generic_xml.py +++ b/onadata/apps/restservice/services/generic_xml.py @@ -1,12 +1,24 @@ +# -*- coding: utf-8 -*- +""" +Post submisison XML data to an external service that accepts an XML post. +""" import requests from onadata.apps.restservice.RestServiceInterface import RestServiceInterface class ServiceDefinition(RestServiceInterface): - id = u'xml' - verbose_name = u'XML POST' + """ + Post submisison XML data to an external service that accepts an XML post. + """ - def send(self, url, submission_instance): + # pylint: disable=invalid-name + id = "xml" + verbose_name = "XML POST" + + def send(self, url, data=None): + """ + Post submisison XML data to an external service that accepts an XML post. + """ headers = {"Content-Type": "application/xml"} - requests.post(url, data=submission_instance.xml, headers=headers) + requests.post(url, data=data.xml, headers=headers) diff --git a/onadata/apps/restservice/services/textit.py b/onadata/apps/restservice/services/textit.py index 427372c0a0..5e6333dd18 100644 --- a/onadata/apps/restservice/services/textit.py +++ b/onadata/apps/restservice/services/textit.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Post submission data to a textit/rapidpro server. +""" import json import requests from six import iteritems @@ -10,19 +14,24 @@ class ServiceDefinition(RestServiceInterface): + """ + Post submission data to a textit/rapidpro server. + """ + + # pylint: disable=invalid-name id = TEXTIT verbose_name = "TextIt POST" - def send(self, url, submission_instance): + def send(self, url, data=None): """ Sends the submission to the configured rest service :param url: - :param submission_instance: + :param data: :return: """ - extra_data = self.clean_keys_of_slashes(submission_instance.json) + extra_data = self.clean_keys_of_slashes(data.json) - data_value = MetaData.textit(submission_instance.xform) + data_value = MetaData.textit(data.xform) if data_value: token, flow, contacts = data_value.split(METADATA_SEPARATOR) @@ -33,7 +42,7 @@ def send(self, url, submission_instance): } headers = { "Content-Type": "application/json", - "Authorization": "Token {}".format(token), + "Authorization": f"Token {token}", } requests.post(url, headers=headers, data=json.dumps(post_data)) diff --git a/onadata/apps/restservice/signals.py b/onadata/apps/restservice/signals.py index 7782abbb17..fea296a59d 100644 --- a/onadata/apps/restservice/signals.py +++ b/onadata/apps/restservice/signals.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ RestService signals module """ @@ -11,11 +11,11 @@ settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False ) -# pylint: disable=C0103 +# pylint: disable=invalid-name trigger_webhook = django.dispatch.Signal(providing_args=["instance"]) -def call_webhooks(sender, **kwargs): # pylint: disable=W0613 +def call_webhooks(sender, **kwargs): # pylint: disable=unused-argument """ Call webhooks signal. """ diff --git a/onadata/apps/restservice/tasks.py b/onadata/apps/restservice/tasks.py index b1895cc2bf..e9acd263d2 100644 --- a/onadata/apps/restservice/tasks.py +++ b/onadata/apps/restservice/tasks.py @@ -1,11 +1,16 @@ +# -*- coding: utf-8 -*- +""" +restservice async functions. +""" +from onadata.apps.logger.models.instance import Instance from onadata.apps.restservice.utils import call_service from onadata.celery import app @app.task() def call_service_async(instance_pk): + """Async function that calls call_service().""" # load the parsed instance - from onadata.apps.logger.models.instance import Instance try: instance = Instance.objects.get(pk=instance_pk) diff --git a/onadata/apps/restservice/tests/test_restservice.py b/onadata/apps/restservice/tests/test_restservice.py index 9c525714f0..21ea036b34 100644 --- a/onadata/apps/restservice/tests/test_restservice.py +++ b/onadata/apps/restservice/tests/test_restservice.py @@ -35,15 +35,18 @@ def setUp(self): self._publish_xls_file(path) self.xform = XForm.objects.all().reverse()[0] - def wait(self, t=1): - time.sleep(t) + # pylint: disable=no-self-use + def wait(self, duration=1): + """Sleep for 1 second or as defined by ``duration``.""" + time.sleep(duration) def _create_rest_service(self): - rs = RestService( + service = RestService( service_url=self.service_url, xform=self.xform, name=self.service_name ) - rs.save() - self.restservice = rs + service.save() + + return service def _add_rest_service(self, service_url, service_name): add_service_url = reverse( @@ -58,7 +61,9 @@ def _add_rest_service(self, service_url, service_name): self.assertEqual(response.status_code, 200) self.assertEqual(RestService.objects.all().count(), count + 1) - def add_rest_service_with_usename_and_id_string_in_uppercase(self): + # pylint: disable=invalid-name + def add_rest_service_with_username_and_id_string_in_uppercase(self): + """Test that the service url is not case sensitive""" add_service_url = reverse( add_service, kwargs={ @@ -70,19 +75,23 @@ def add_rest_service_with_usename_and_id_string_in_uppercase(self): self.assertEqual(response.status_code, 200) def test_create_rest_service(self): + """Test the RestService model.""" count = RestService.objects.all().count() self._create_rest_service() self.assertEqual(RestService.objects.all().count(), count + 1) def test_service_definition(self): - self._create_rest_service() - sv = self.restservice.get_service_definition()() - self.assertEqual(isinstance(sv, RestServiceInterface), True) + """Test the service_definition is an instance of RestServiceInterface""" + restservice = self._create_rest_service() + service = restservice.get_service_definition()() + self.assertEqual(isinstance(service, RestServiceInterface), True) def test_add_service(self): + """Test adding a restservice.""" self._add_rest_service(self.service_url, self.service_name) def test_anon_service_view(self): + """Test the rest service section is not available to asynchronous users.""" self.xform.shared = True self.xform.save() self._logout() @@ -101,6 +110,7 @@ def test_anon_service_view(self): ) def test_delete_service(self): + """Test deletion of a service.""" self._add_rest_service(self.service_url, self.service_name) count = RestService.objects.all().count() service = RestService.objects.reverse()[0] @@ -113,7 +123,9 @@ def test_delete_service(self): self.assertEqual(response.status_code, 200) self.assertEqual(RestService.objects.all().count(), count - 1) + # pylint: disable=invalid-name def test_add_rest_service_with_wrong_id_string(self): + """Test the id_string is validated when adding a service url.""" add_service_url = reverse( add_service, kwargs={"username": self.user.username, "id_string": "wrong_id_string"}, @@ -125,6 +137,7 @@ def test_add_rest_service_with_wrong_id_string(self): @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch("requests.post") def test_textit_service(self, mock_http): + """Test the textit service.""" service_url = "https://textit.io/api/v1/runs.json" service_name = "textit" @@ -137,7 +150,7 @@ def test_textit_service(self, mock_http): MetaData.textit( self.xform, - data_value="{}|{}|{}".format(api_token, flow_uuid, default_contact), + data_value=f"{api_token}|{flow_uuid}|{default_contact}", ) xml_submission = os.path.join( @@ -152,6 +165,7 @@ def test_textit_service(self, mock_http): @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch("requests.post") def test_rest_service_not_set(self, mock_http): + """Test a requests.post is not called when a service is not defined.""" xml_submission = os.path.join( self.this_directory, "fixtures", "dhisform_submission1.xml" ) @@ -162,6 +176,7 @@ def test_rest_service_not_set(self, mock_http): self.assertEqual(mock_http.call_count, 0) def test_clean_keys_of_slashes(self): + """Test ServiceDefinition.clean_keys_of_slashes() function.""" service = ServiceDefinition() test_data = { diff --git a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py index 1900e12930..19ad06421a 100644 --- a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py +++ b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py @@ -18,7 +18,7 @@ class TestRestServicesViewSet(TestAbstractViewSet): """ def setUp(self): - super(TestRestServicesViewSet, self).setUp() + super().setUp() self.view = RestServicesViewSet.as_view( { "delete": "destroy", @@ -31,6 +31,7 @@ def setUp(self): self._publish_xls_form_to_project() def test_create(self): + """Test create service via API.""" count = RestService.objects.all().count() post_data = { @@ -44,7 +45,9 @@ def test_create(self): self.assertEqual(response.status_code, 201) self.assertEqual(count + 1, RestService.objects.all().count()) + # pylint: disable=invalid-name def test_textit_service_missing_params(self): + """Test creating a service with a missing parameter fails.""" post_data = { "name": "textit", "service_url": "https://textit.io", @@ -74,7 +77,7 @@ def _create_textit_service(self): meta = MetaData.objects.filter(object_id=self.xform.id, data_type="textit") self.assertEqual(len(meta), 1) - rs = RestService.objects.last() + service = RestService.objects.last() expected_dict = { "name": "textit", @@ -82,7 +85,7 @@ def _create_textit_service(self): "auth_token": "sadsdfhsdf", "flow_uuid": "sdfskhfskdjhfs", "service_url": "https://textit.io", - "id": rs.pk, + "id": service.pk, "xform": self.xform.pk, "active": True, "inactive_reason": "", @@ -96,9 +99,11 @@ def _create_textit_service(self): return response.data def test_create_textit_service(self): + """Test creating textit service via API.""" self._create_textit_service() def test_retrieve_textit_services(self): + """Test retrieving the textit service via API.""" response_data = self._create_textit_service() _id = response_data.get("id") @@ -123,6 +128,7 @@ def test_retrieve_textit_services(self): self.assertEqual(response.data, expected_dict) def test_delete_textit_service(self): + """Test deleting a textit service via API""" rest = self._create_textit_service() count = RestService.objects.all().count() meta_count = MetaData.objects.filter( @@ -140,6 +146,7 @@ def test_delete_textit_service(self): self.assertEqual(meta_count - 1, a_meta_count) def test_update(self): + """Test updating a service via API.""" rest = RestService( name="testservice", service_url="http://serviec.io", xform=self.xform ) @@ -181,9 +188,10 @@ def test_update(self): self.assertEqual(MetaData.objects.count(), metadata_count) def test_update_with_errors(self): + """Test update errors if records is not in the write format.""" rest = self._create_textit_service() - data_value = "{}|{}".format("test", "test2") + data_value = "test|test2" MetaData.textit(self.xform, data_value) request = self.factory.get("/", **self.extra) @@ -214,6 +222,7 @@ def test_update_with_errors(self): self.assertEqual(response.status_code, 200) def test_delete(self): + """Test delete service via API.""" rest = RestService( name="testservice", service_url="http://serviec.io", xform=self.xform ) @@ -228,6 +237,7 @@ def test_delete(self): self.assertEqual(count - 1, RestService.objects.all().count()) def test_retrieve(self): + """Test retrieving a service via API.""" rest = RestService( name="testservice", service_url="http://serviec.io", xform=self.xform ) @@ -251,6 +261,7 @@ def test_retrieve(self): self.assertEqual(response.data, data) def test_duplicate_rest_service(self): + """Test duplicate service is not created.""" post_data = { "name": "textit", "service_url": "https://textit.io", @@ -280,6 +291,7 @@ def test_duplicate_rest_service(self): @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @patch("requests.post") def test_textit_flow(self, mock_http): + """Test posting a submission to textit service.""" rest = RestService( name="textit", service_url="https://server.io", xform=self.xform ) @@ -287,9 +299,7 @@ def test_textit_flow(self, mock_http): MetaData.textit( self.xform, - data_value="{}|{}|{}".format( - "sadsdfhsdf", "sdfskhfskdjhfs", "ksadaskjdajsda" - ), + data_value="sadsdfhsdf|sdfskhfskdjhfs|ksadaskjdajsda", ) self.assertFalse(mock_http.called) @@ -298,25 +308,8 @@ def test_textit_flow(self, mock_http): self.assertTrue(mock_http.called) self.assertEqual(mock_http.call_count, 4) - @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch("requests.post") - def test_textit_flow_without_parsed_instances(self, mock_http): - rest = RestService( - name="textit", service_url="https://server.io", xform=self.xform - ) - rest.save() - - MetaData.textit( - self.xform, - data_value="{}|{}|{}".format( - "sadsdfhsdf", "sdfskhfskdjhfs", "ksadaskjdajsda" - ), - ) - self.assertFalse(mock_http.called) - self._make_submissions() - self.assertTrue(mock_http.called) - def test_create_rest_service_invalid_form_id(self): + """Test creating with an invalid form id fails.""" count = RestService.objects.all().count() post_data = { diff --git a/onadata/apps/restservice/utils.py b/onadata/apps/restservice/utils.py index ff1c9caf68..795157968e 100644 --- a/onadata/apps/restservice/utils.py +++ b/onadata/apps/restservice/utils.py @@ -1,21 +1,27 @@ +# -*- coding: utf-8 -*- +""" +restservice utility functions. +""" import logging - -from django.utils.translation import gettext as _ +import sys from onadata.apps.restservice.models import RestService from onadata.libs.utils.common_tags import GOOGLE_SHEET +from onadata.libs.utils.common_tools import report_exception def call_service(submission_instance): + """Sends submissions to linked services.""" # lookup service which is not google sheet service services = RestService.objects.filter( - xform_id=submission_instance.xform_id).exclude(name=GOOGLE_SHEET) + xform_id=submission_instance.xform_id + ).exclude(name=GOOGLE_SHEET) # call service send with url and data parameters - for sv in services: - # TODO: Queue service + for service_def in services: + # pylint: disable=broad-except try: - service = sv.get_service_definition()() - service.send(sv.service_url, submission_instance) + service = service_def.get_service_definition()() + service.send(service_def.service_url, submission_instance) except Exception as e: - # TODO: Handle gracefully | requeue/resend - logging.exception(_(u'Service threw exception: %s' % str(e))) + report_exception(f"Service call failed: {e}", e, sys.exc_info()) + logging.exception("Service threw exception: %s", e) diff --git a/onadata/apps/restservice/views.py b/onadata/apps/restservice/views.py index 194935e8f0..e8e1a4c5c3 100644 --- a/onadata/apps/restservice/views.py +++ b/onadata/apps/restservice/views.py @@ -1,87 +1,102 @@ -import json +# -*- coding: utf-8 -*- +""" +restservice views. +""" from django.contrib.auth.decorators import login_required from django.db.utils import IntegrityError -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.shortcuts import render from django.template.base import Template from django.template.context import Context from django.template.loader import render_to_string from django.utils.translation import gettext as _ -from onadata.libs.utils.viewer_tools import get_form from onadata.apps.restservice.forms import RestServiceForm from onadata.apps.restservice.models import RestService +from onadata.libs.utils.viewer_tools import get_form @login_required def add_service(request, username, id_string): + """Add a service.""" data = {} form = RestServiceForm() - xform_kwargs = { - 'id_string__iexact': id_string, - 'user__username__iexact': username - } - xform = get_form(xform_kwargs) - if request.method == 'POST': + xform = get_form( + {"id_string__iexact": id_string, "user__username__iexact": username} + ) + if request.method == "POST": form = RestServiceForm(request.POST) restservice = None if form.is_valid(): - service_name = form.cleaned_data['service_name'] - service_url = form.cleaned_data['service_url'] + service_name = form.cleaned_data["service_name"] + service_url = form.cleaned_data["service_url"] try: - rs = RestService(service_url=service_url, - name=service_name, xform=xform) - rs.save() + service = RestService( + service_url=service_url, name=service_name, xform=xform + ) + service.save() except IntegrityError: - message = _(u"Service already defined.") - status = 'fail' + message = _("Service already defined.") + status = "fail" else: - status = 'success' - message = (_(u"Successfully added service %(name)s.") - % {'name': service_name}) - service_tpl = render_to_string("service.html", { - "sv": rs, "username": xform.user.username, - "id_string": xform.id_string}) + status = "success" + message = _("Successfully added service %(name)s.") % { + "name": service_name + } + service_tpl = render_to_string( + "service.html", + { + "sv": service, + "username": xform.user.username, + "id_string": xform.id_string, + }, + ) restservice = service_tpl else: - status = 'fail' - message = _(u"Please fill in all required fields") + status = "fail" + message = _("Please fill in all required fields") if form.errors: for field in form: - message += Template(u"{{ field.errors }}")\ - .render(Context({'field': field})) + message += Template("{{ field.errors }}").render( + Context({"field": field}) + ) if request.is_ajax(): - response = {'status': status, 'message': message} + response = {"status": status, "message": message} if restservice: - response["restservice"] = u"%s" % restservice + response["restservice"] = f"{restservice}" - return HttpResponse(json.dumps(response)) + return JsonResponse(response) - data['status'] = status - data['message'] = message + data["status"] = status + data["message"] = message - data['list_services'] = RestService.objects.filter(xform=xform) - data['form'] = form - data['username'] = username - data['id_string'] = id_string + data["list_services"] = RestService.objects.filter(xform=xform) + data["form"] = form + data["username"] = username + data["id_string"] = id_string return render(request, "add-service.html", data) +@login_required def delete_service(request, username, id_string): + """Delete a service view.""" success = "FAILED" - if request.method == 'POST': - pk = request.POST.get('service-id') - if pk: + xform = get_form( + {"id_string__iexact": id_string, "user__username__iexact": username} + ) + if request.method == "POST": + service_id = request.POST.get("service-id") + if service_id: try: - rs = RestService.objects.get(pk=int(pk)) + service = RestService.objects.get(pk=int(service_id), xform=xform) except RestService.DoesNotExist: pass else: - rs.delete() + service.delete() success = "OK" return HttpResponse(success) diff --git a/onadata/apps/restservice/viewsets/restservices_viewset.py b/onadata/apps/restservice/viewsets/restservices_viewset.py index 932aa8db56..b0862920c1 100644 --- a/onadata/apps/restservice/viewsets/restservices_viewset.py +++ b/onadata/apps/restservice/viewsets/restservices_viewset.py @@ -1,28 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Implements the /api/v1/restservices endpoint. +""" from django.conf import settings from django.utils.module_loading import import_string -from rest_framework.viewsets import ModelViewSet + from rest_framework.response import Response +from rest_framework.viewsets import ModelViewSet from onadata.apps.api.permissions import RestServiceObjectPermissions -from onadata.libs.serializers.textit_serializer import TextItSerializer +from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.restservice.models import RestService from onadata.libs import filters -from onadata.libs.serializers.restservices_serializer import \ - RestServiceSerializer -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.last_modified_mixin import LastModifiedMixin +from onadata.libs.serializers.restservices_serializer import RestServiceSerializer +from onadata.libs.serializers.textit_serializer import TextItSerializer from onadata.libs.utils.common_tags import TEXTIT -from onadata.apps.api.tools import get_baseviewset_class - +# pylint: disable=invalid-name BaseViewset = get_baseviewset_class() def get_serializer_class(name): - services_to_serializers = getattr(settings, - 'REST_SERVICES_TO_SERIALIZERS', {}) + """Returns a serilizer class with the given ``name``.""" + services_to_serializers = getattr(settings, "REST_SERVICES_TO_SERIALIZERS", {}) serializer_class = services_to_serializers.get(name) if serializer_class: @@ -31,27 +34,36 @@ def get_serializer_class(name): if name == TEXTIT: return TextItSerializer + return RestServiceSerializer + -class RestServicesViewSet(AuthenticateHeaderMixin, - CacheControlMixin, LastModifiedMixin, BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class RestServicesViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + LastModifiedMixin, + BaseViewset, + ModelViewSet, +): """ This endpoint provides access to form rest services. """ - queryset = RestService.objects.select_related('xform') + queryset = RestService.objects.select_related("xform") serializer_class = RestServiceSerializer - permission_classes = [RestServiceObjectPermissions, ] - filter_backends = (filters.RestServiceFilter, ) + permission_classes = [ + RestServiceObjectPermissions, + ] + filter_backends = (filters.RestServiceFilter,) def get_serializer_class(self): - name = self.request.data.get('name') + name = self.request.data.get("name") serializer_class = get_serializer_class(name) if serializer_class: return serializer_class - return super(RestServicesViewSet, self).get_serializer_class() + return super().get_serializer_class() def retrieve(self, request, *args, **kwargs): instance = self.get_object() From 793769b84c13aee7ce106d95bc1defb03a86c0d6 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 12:34:32 +0300 Subject: [PATCH 197/234] Check if xform.json is a dict --- .../apps/api/viewsets/merged_xform_viewset.py | 53 +-- .../apps/api/viewsets/open_data_viewset.py | 168 ++++----- .../apps/api/viewsets/v2/tableau_viewset.py | 199 ++++++----- onadata/apps/logger/models/xform.py | 23 +- onadata/apps/main/views.py | 4 +- onadata/apps/sms_support/autodoc.py | 325 +++++++++--------- onadata/apps/sms_support/parser.py | 308 +++++++++-------- onadata/apps/viewer/models/data_dictionary.py | 2 +- onadata/apps/viewer/models/parsed_instance.py | 2 +- .../libs/serializers/attachment_serializer.py | 2 +- .../serializers/merged_xform_serializer.py | 168 +++++---- onadata/libs/utils/csv_import.py | 5 +- 12 files changed, 672 insertions(+), 587 deletions(-) diff --git a/onadata/apps/api/viewsets/merged_xform_viewset.py b/onadata/apps/api/viewsets/merged_xform_viewset.py index af3470ae02..fb4d9bcaa0 100644 --- a/onadata/apps/api/viewsets/merged_xform_viewset.py +++ b/onadata/apps/api/viewsets/merged_xform_viewset.py @@ -16,53 +16,62 @@ from onadata.apps.logger.models import Instance, MergedXForm from onadata.libs import filters from onadata.libs.renderers import renderers -from onadata.libs.serializers.merged_xform_serializer import \ - MergedXFormSerializer +from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer # pylint: disable=too-many-ancestors -class MergedXFormViewSet(mixins.CreateModelMixin, - mixins.DestroyModelMixin, - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet): +class MergedXFormViewSet( + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): """ Merged XForms viewset: create, list, retrieve, destroy """ - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.PublicDatasetsFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.PublicDatasetsFilter, + ) permission_classes = [XFormPermissions] - queryset = MergedXForm.objects.filter(deleted_at__isnull=True).annotate( - number_of_submissions=Sum('xforms__num_of_submissions')).all() + queryset = ( + MergedXForm.objects.filter(deleted_at__isnull=True) + .annotate(number_of_submissions=Sum("xforms__num_of_submissions")) + .all() + ) serializer_class = MergedXFormSerializer - renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + \ - [renderers.StaticXMLRenderer] + renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ + renderers.StaticXMLRenderer + ] # pylint: disable=unused-argument - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def form(self, *args, **kwargs): """Return XForm JSON, XLS or XML representing""" - fmt = kwargs['format'] - if fmt not in ['json', 'xml', 'xls']: + fmt = kwargs["format"] + if fmt not in ["json", "xml", "xls"]: return HttpResponseBadRequest( - '400 BAD REQUEST', content_type='application/json', status=400) + "400 BAD REQUEST", content_type="application/json", status=400 + ) merged_xform = self.get_object() data = getattr(merged_xform, fmt) - if fmt == 'json': - data = json.loads(data) + if fmt == "json": + data = json.loads(data) if isinstance(data, str) else data return Response(data) # pylint: disable=unused-argument - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, *args, **kwargs): """Return data from the merged xforms""" merged_xform = self.get_object() queryset = Instance.objects.filter( - xform__in=merged_xform.xforms.all()).order_by('pk') + xform__in=merged_xform.xforms.all() + ).order_by("pk") - return Response(queryset.values_list('json', flat=True)) + return Response(queryset.values_list("json", flat=True)) diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index c6ebcd24bc..846fee3db2 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -33,16 +33,17 @@ GEOLOCATION, MULTIPLE_SELECT_TYPE, REPEAT_SELECT_TYPE, - NA_REP) + NA_REP, +) BaseViewset = get_baseviewset_class() -IGNORED_FIELD_TYPES = ['select one', 'select multiple'] +IGNORED_FIELD_TYPES = ["select one", "select multiple"] # index tags -DEFAULT_OPEN_TAG = '[' -DEFAULT_CLOSE_TAG = ']' +DEFAULT_OPEN_TAG = "[" +DEFAULT_CLOSE_TAG = "]" DEFAULT_INDEX_TAGS = (DEFAULT_OPEN_TAG, DEFAULT_CLOSE_TAG) -DEFAULT_NA_REP = getattr(settings, 'NA_REP', NA_REP) +DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) def replace_special_characters_with_underscores(data): @@ -57,9 +58,9 @@ def process_tableau_data(data, xform): """ def get_xpath(key, nested_key): - val = nested_key.split('/') - nested_key_diff = val[len(key.split('/')):] - xpaths = key + f'[{index}]/' + '/'.join(nested_key_diff) + val = nested_key.split("/") + nested_key_diff = val[len(key.split("/")) :] + xpaths = key + f"[{index}]/" + "/".join(nested_key_diff) return xpaths def get_updated_data_dict(key, value, data_dict): @@ -72,7 +73,7 @@ def get_updated_data_dict(key, value, data_dict): if isinstance(value, str) and data_dict: choices = value.split(" ") for choice in choices: - xpaths = f'{key}/{choice}' + xpaths = f"{key}/{choice}" data_dict[xpaths] = choice elif isinstance(value, list): try: @@ -106,11 +107,9 @@ def get_ordered_repeat_value(key, item, index): qstn_type = xform.get_element(nested_key).type xpaths = get_xpath(key, nested_key) if qstn_type == MULTIPLE_SELECT_TYPE: - data = get_updated_data_dict( - xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data) elif qstn_type == REPEAT_SELECT_TYPE: - data = get_updated_data_dict( - xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data) else: data[xpaths] = nested_val return data @@ -124,7 +123,10 @@ def get_ordered_repeat_value(key, item, index): flat_dict = dict.fromkeys(diff, None) for (key, value) in row.items(): if isinstance(value, list) and key not in [ - ATTACHMENTS, NOTES, GEOLOCATION]: + ATTACHMENTS, + NOTES, + GEOLOCATION, + ]: for index, item in enumerate(value, start=1): # order repeat according to xform order item = get_ordered_repeat_value(key, item, index) @@ -133,15 +135,13 @@ def get_ordered_repeat_value(key, item, index): try: qstn_type = xform.get_element(key).type if qstn_type == MULTIPLE_SELECT_TYPE: - flat_dict = get_updated_data_dict( - key, value, flat_dict) - if qstn_type == 'geopoint': - parts = value.split(' ') - gps_xpaths = \ - DataDictionary.get_additional_geopoint_xpaths( - key) - gps_parts = dict( - [(xpath, None) for xpath in gps_xpaths]) + flat_dict = get_updated_data_dict(key, value, flat_dict) + if qstn_type == "geopoint": + parts = value.split(" ") + gps_xpaths = DataDictionary.get_additional_geopoint_xpaths( + key + ) + gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) if len(parts) == 4: gps_parts = dict(zip(gps_xpaths, parts)) flat_dict.update(gps_parts) @@ -154,58 +154,55 @@ def get_ordered_repeat_value(key, item, index): return result -class OpenDataViewSet(ETagsMixin, CacheControlMixin, - BaseViewset, ModelViewSet): - permission_classes = (OpenDataViewSetPermissions, ) +class OpenDataViewSet(ETagsMixin, CacheControlMixin, BaseViewset, ModelViewSet): + permission_classes = (OpenDataViewSetPermissions,) queryset = OpenData.objects.filter() - lookup_field = 'uuid' + lookup_field = "uuid" serializer_class = OpenDataSerializer flattened_dict = {} MAX_INSTANCES_PER_REQUEST = 1000 pagination_class = StandardPageNumberPagination def get_tableau_type(self, xform_type): - ''' + """ Returns a tableau-supported type based on a xform type. - ''' + """ tableau_types = { - 'integer': 'int', - 'decimal': 'float', - 'dateTime': 'datetime', - 'text': 'string' + "integer": "int", + "decimal": "float", + "dateTime": "datetime", + "text": "string", } - return tableau_types.get(xform_type, 'string') + return tableau_types.get(xform_type, "string") def flatten_xform_columns(self, json_of_columns_fields): - ''' + """ Flattens a json of column fields and the result is set to a class variable. - ''' + """ for a in json_of_columns_fields: - self.flattened_dict[a.get('name')] = self.get_tableau_type( - a.get('type')) + self.flattened_dict[a.get("name")] = self.get_tableau_type(a.get("type")) # using IGNORED_FIELD_TYPES so that choice values are not included. - if a.get('children') and a.get('type') not in IGNORED_FIELD_TYPES: - self.flatten_xform_columns(a.get('children')) + if a.get("children") and a.get("type") not in IGNORED_FIELD_TYPES: + self.flatten_xform_columns(a.get("children")) def get_tableau_column_headers(self): - ''' + """ Retrieve columns headers that are valid in tableau. - ''' + """ tableau_colulmn_headers = [] def append_to_tableau_colulmn_headers(header, question_type=None): - quest_type = 'string' + quest_type = "string" if question_type: quest_type = question_type # alias can be updated in the future to question labels - tableau_colulmn_headers.append({ - 'id': header, - 'dataType': quest_type, - 'alias': header - }) + tableau_colulmn_headers.append( + {"id": header, "dataType": quest_type, "alias": header} + ) + # Remove metadata fields from the column headers # Calling set to remove duplicates in group data xform_headers = set(remove_metadata_fields(self.xform_headers)) @@ -214,30 +211,30 @@ def append_to_tableau_colulmn_headers(header, question_type=None): # tableau. for header in xform_headers: for quest_name, quest_type in self.flattened_dict.items(): - if header == quest_name or header.endswith('_%s' % quest_name): + if header == quest_name or header.endswith("_%s" % quest_name): append_to_tableau_colulmn_headers(header, quest_type) break else: - if header == '_id': + if header == "_id": append_to_tableau_colulmn_headers(header, "int") else: append_to_tableau_colulmn_headers(header) return tableau_colulmn_headers - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, **kwargs): """ Streams submission data response matching uuid in the request. """ self.object = self.get_object() # get greater than value and cast it to an int - gt_id = request.query_params.get('gt_id') + gt_id = request.query_params.get("gt_id") gt_id = gt_id and parse_int(gt_id) - count = request.query_params.get('count') + count = request.query_params.get("count") pagination_keys = [ self.paginator.page_query_param, - self.paginator.page_size_query_param + self.paginator.page_size_query_param, ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) @@ -249,25 +246,30 @@ def data(self, request, **kwargs): xform = self.object.content_object if xform.is_merged_dataset: - qs_kwargs = {'xform_id__in': list( - xform.mergedxform.xforms.values_list('pk', flat=True))} + qs_kwargs = { + "xform_id__in": list( + xform.mergedxform.xforms.values_list("pk", flat=True) + ) + } else: - qs_kwargs = {'xform_id': xform.pk} + qs_kwargs = {"xform_id": xform.pk} if gt_id: - qs_kwargs.update({'id__gt': gt_id}) + qs_kwargs.update({"id__gt": gt_id}) # Filter out deleted submissions instances = Instance.objects.filter( - **qs_kwargs, deleted_at__isnull=True).order_by('pk') + **qs_kwargs, deleted_at__isnull=True + ).order_by("pk") if count: - return Response({'count': instances.count()}) + return Response({"count": instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) data = process_tableau_data( - TableauDataSerializer(instances, many=True).data, xform) + TableauDataSerializer(instances, many=True).data, xform + ) return self.get_streaming_response(data) @@ -277,12 +279,10 @@ def get_streaming_response(self, data): """Get a StreamingHttpResponse response object""" def get_json_string(item): - return json.dumps({ - re.sub(r"\W", r"_", a): b for a, b in item.items()}) + return json.dumps({re.sub(r"\W", r"_", a): b for a, b in item.items()}) response = StreamingHttpResponse( - json_stream(data, get_json_string), - content_type="application/json" + json_stream(data, get_json_string), content_type="application/json" ) # set headers on streaming response @@ -295,54 +295,56 @@ def destroy(self, request, *args, **kwargs): self.get_object().delete() return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): self.object = self.get_object() if isinstance(self.object.content_object, XForm): xform = self.object.content_object headers = xform.get_headers() - self.xform_headers = replace_special_characters_with_underscores( - headers) + self.xform_headers = replace_special_characters_with_underscores(headers) - xform_json = json.loads(xform.json) + xform_json = xform.json_dict() self.flatten_xform_columns( - json_of_columns_fields=xform_json.get('children')) + json_of_columns_fields=xform_json.get("children") + ) tableau_column_headers = self.get_tableau_column_headers() data = { - 'column_headers': tableau_column_headers, - 'connection_name': "%s_%s" % (xform.project_id, - xform.id_string), - 'table_alias': xform.title + "column_headers": tableau_column_headers, + "connection_name": "%s_%s" % (xform.project_id, xform.id_string), + "table_alias": xform.title, } return Response(data=data, status=status.HTTP_200_OK) return Response(status=status.HTTP_404_NOT_FOUND) - @action(methods=['GET'], detail=False) + @action(methods=["GET"], detail=False) def uuid(self, request, *args, **kwargs): - data_type = request.query_params.get('data_type') - object_id = request.query_params.get('object_id') + data_type = request.query_params.get("data_type") + object_id = request.query_params.get("object_id") if not data_type or not object_id: return Response( data="Query params data_type and object_id are required", - status=status.HTTP_400_BAD_REQUEST) + status=status.HTTP_400_BAD_REQUEST, + ) - if data_type == 'xform': + if data_type == "xform": xform = get_object_or_404(XForm, id=object_id) if request.user.has_perm("change_xform", xform): ct = ContentType.objects.get_for_model(xform) _open_data = get_object_or_404( - OpenData, object_id=object_id, content_type=ct) + OpenData, object_id=object_id, content_type=ct + ) if _open_data: return Response( - data={'uuid': _open_data.uuid}, - status=status.HTTP_200_OK) + data={"uuid": _open_data.uuid}, status=status.HTTP_200_OK + ) else: raise PermissionDenied( - _((u"You do not have permission to perform this action."))) + _(("You do not have permission to perform this action.")) + ) return Response(status=status.HTTP_404_NOT_FOUND) diff --git a/onadata/apps/api/viewsets/v2/tableau_viewset.py b/onadata/apps/api/viewsets/v2/tableau_viewset.py index 80ad85a0e2..158d0ce055 100644 --- a/onadata/apps/api/viewsets/v2/tableau_viewset.py +++ b/onadata/apps/api/viewsets/v2/tableau_viewset.py @@ -1,4 +1,7 @@ -import json +# -*- coding: utf-8 -*- +""" +Implements the /api/v2/tableau endpoint +""" import re from typing import List @@ -12,27 +15,33 @@ from onadata.apps.logger.models import Instance from onadata.apps.logger.models.xform import XForm from onadata.apps.api.tools import replace_attachment_name_with_url -from onadata.apps.api.viewsets.open_data_viewset import ( - OpenDataViewSet) +from onadata.apps.api.viewsets.open_data_viewset import OpenDataViewSet from onadata.libs.serializers.data_serializer import TableauDataSerializer from onadata.libs.utils.common_tags import ( - ID, MULTIPLE_SELECT_TYPE, REPEAT_SELECT_TYPE, PARENT_TABLE, PARENT_ID) + ID, + MULTIPLE_SELECT_TYPE, + REPEAT_SELECT_TYPE, + PARENT_TABLE, + PARENT_ID, +) -DEFAULT_TABLE_NAME = 'data' -GPS_PARTS = ['latitude', 'longitude', 'altitude', 'precision'] +DEFAULT_TABLE_NAME = "data" +GPS_PARTS = ["latitude", "longitude", "altitude", "precision"] def process_tableau_data( - data, xform, - parent_table: str = None, - parent_id: int = None, - current_table: str = DEFAULT_TABLE_NAME): + data, + xform, + parent_table: str = None, + parent_id: int = None, + current_table: str = DEFAULT_TABLE_NAME, +): result = [] if data: for idx, row in enumerate(data, start=1): flat_dict = defaultdict(list) - row_id = row.get('_id') + row_id = row.get("_id") if not row_id and parent_id: row_id = int(pairing(parent_id, idx)) @@ -45,35 +54,38 @@ def process_tableau_data( for (key, value) in row.items(): qstn = xform.get_element(key) if qstn: - qstn_type = qstn.get('type') - qstn_name = qstn.get('name') + qstn_type = qstn.get("type") + qstn_name = qstn.get("name") prefix_parts = [ - question['name'] for question in qstn.get_lineage() - if question['type'] == 'group' + question["name"] + for question in qstn.get_lineage() + if question["type"] == "group" ] prefix = "_".join(prefix_parts) if qstn_type == REPEAT_SELECT_TYPE: repeat_data = process_tableau_data( - value, xform, + value, + xform, parent_table=current_table, parent_id=row_id, - current_table=qstn_name) - cleaned_data = unpack_repeat_data( - repeat_data, flat_dict) + current_table=qstn_name, + ) + cleaned_data = unpack_repeat_data(repeat_data, flat_dict) flat_dict[qstn_name] = cleaned_data elif qstn_type == MULTIPLE_SELECT_TYPE: picked_choices = value.split(" ") choice_names = [ - question["name"] for question in qstn["children"]] - list_name = qstn.get('list_name') + question["name"] for question in qstn["children"] + ] + list_name = qstn.get("list_name") select_multiple_data = unpack_select_multiple_data( - picked_choices, list_name, choice_names, prefix) + picked_choices, list_name, choice_names, prefix + ) flat_dict.update(select_multiple_data) - elif qstn_type == 'geopoint': - gps_parts = unpack_gps_data( - value, qstn_name, prefix) + elif qstn_type == "geopoint": + gps_parts = unpack_gps_data(value, qstn_name, prefix) flat_dict.update(gps_parts) else: if prefix: @@ -83,14 +95,13 @@ def process_tableau_data( return result -def unpack_select_multiple_data(picked_choices, list_name, - choice_names, prefix): +def unpack_select_multiple_data(picked_choices, list_name, choice_names, prefix): unpacked_data = {} for choice in choice_names: qstn_name = f"{list_name}_{choice}" if prefix: - qstn_name = prefix + '_' + qstn_name + qstn_name = prefix + "_" + qstn_name if choice in picked_choices: unpacked_data[qstn_name] = "TRUE" @@ -117,16 +128,15 @@ def unpack_repeat_data(repeat_data, flat_dict): def unpack_gps_data(value, qstn_name, prefix): - value_parts = value.split(' ') + value_parts = value.split(" ") gps_xpath_parts = [] for part in GPS_PARTS: name = f"_{qstn_name}_{part}" if prefix: - name = prefix + '_' + name + name = prefix + "_" + name gps_xpath_parts.append((name, None)) if len(value_parts) == 4: - gps_parts = dict( - zip(dict(gps_xpath_parts), value_parts)) + gps_parts = dict(zip(dict(gps_xpath_parts), value_parts)) return gps_parts @@ -135,9 +145,9 @@ def clean_xform_headers(headers: list) -> list: for header in headers: if re.search(r"\[+\d+\]", header): repeat_count = len(re.findall(r"\[+\d+\]", header)) - header = header.split('/')[repeat_count].replace('[1]', '') + header = header.split("/")[repeat_count].replace("[1]", "") - if not header.endswith('gps'): + if not header.endswith("gps"): # Replace special character with underscore header = re.sub(r"\W", r"_", header) ret.append(header) @@ -145,16 +155,16 @@ def clean_xform_headers(headers: list) -> list: class TableauViewSet(OpenDataViewSet): - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def data(self, request, **kwargs): self.object = self.get_object() # get greater than value and cast it to an int - gt_id = request.query_params.get('gt_id') + gt_id = request.query_params.get("gt_id") gt_id = gt_id and parse_int(gt_id) - count = request.query_params.get('count') + count = request.query_params.get("count") pagination_keys = [ self.paginator.page_query_param, - self.paginator.page_size_query_param + self.paginator.page_size_query_param, ] query_param_keys = request.query_params should_paginate = any([k in query_param_keys for k in pagination_keys]) @@ -166,19 +176,23 @@ def data(self, request, **kwargs): xform = self.object.content_object if xform.is_merged_dataset: - qs_kwargs = {'xform_id__in': list( - xform.mergedxform.xforms.values_list('pk', flat=True))} + qs_kwargs = { + "xform_id__in": list( + xform.mergedxform.xforms.values_list("pk", flat=True) + ) + } else: - qs_kwargs = {'xform_id': xform.pk} + qs_kwargs = {"xform_id": xform.pk} if gt_id: - qs_kwargs.update({'id__gt': gt_id}) + qs_kwargs.update({"id__gt": gt_id}) # Filter out deleted submissions instances = Instance.objects.filter( - **qs_kwargs, deleted_at__isnull=True).order_by('pk') + **qs_kwargs, deleted_at__isnull=True + ).order_by("pk") if count: - return Response({'count': instances.count()}) + return Response({"count": instances.count()}) if should_paginate: instances = self.paginate_queryset(instances) @@ -186,7 +200,8 @@ def data(self, request, **kwargs): data = replace_attachment_name_with_url(instances) data = process_tableau_data( - TableauDataSerializer(data, many=True).data, xform) + TableauDataSerializer(data, many=True).data, xform + ) return self.get_streaming_response(data) @@ -194,53 +209,52 @@ def data(self, request, **kwargs): # pylint: disable=arguments-differ def flatten_xform_columns( - self, json_of_columns_fields, table: str = None, - field_prefix: str = None): - ''' + self, json_of_columns_fields, table: str = None, field_prefix: str = None + ): + """ Flattens a json of column fields while splitting columns into separate table names for each repeat - ''' + """ ret = defaultdict(list) for field in json_of_columns_fields: table_name = table or DEFAULT_TABLE_NAME prefix = field_prefix or "" - field_type = field.get('type') + field_type = field.get("type") - if field_type in [REPEAT_SELECT_TYPE, 'group']: - if field_type == 'repeat': - table_name = field.get('name') + if field_type in [REPEAT_SELECT_TYPE, "group"]: + if field_type == "repeat": + table_name = field.get("name") else: prefix = prefix + f"{field['name']}_" columns = self.flatten_xform_columns( - field.get('children'), table=table_name, - field_prefix=prefix) + field.get("children"), table=table_name, field_prefix=prefix + ) for key in columns.keys(): ret[key].extend(columns[key]) elif field_type == MULTIPLE_SELECT_TYPE: - for option in field.get('children'): - list_name = field.get('list_name') - option_name = option.get('name') + for option in field.get("children"): + list_name = field.get("list_name") + option_name = option.get("name") ret[table_name].append( { - 'name': f'{prefix}{list_name}_{option_name}', - 'type': self.get_tableau_type('text') + "name": f"{prefix}{list_name}_{option_name}", + "type": self.get_tableau_type("text"), } ) - elif field_type == 'geopoint': + elif field_type == "geopoint": for part in GPS_PARTS: name = f'_{field["name"]}_{part}' if prefix: name = prefix + name - ret[table_name].append({ - 'name': name, - 'type': self.get_tableau_type(field.get('type')) - }) + ret[table_name].append( + {"name": name, "type": self.get_tableau_type(field.get("type"))} + ) else: ret[table_name].append( { - 'name': prefix + field.get('name'), - 'type': self.get_tableau_type(field.get('type')) + "name": prefix + field.get("name"), + "type": self.get_tableau_type(field.get("type")), } ) return ret @@ -252,29 +266,25 @@ def get_tableau_column_headers(self): tableau_column_headers = defaultdict(list) for table, columns in self.flattened_dict.items(): # Add ID Fields - tableau_column_headers[table].append({ - 'id': ID, - 'dataType': 'int', - 'alias': ID - }) + tableau_column_headers[table].append( + {"id": ID, "dataType": "int", "alias": ID} + ) if table != DEFAULT_TABLE_NAME: - tableau_column_headers[table].append({ - 'id': PARENT_ID, - 'dataType': 'int', - 'alias': PARENT_ID - }) - tableau_column_headers[table].append({ - 'id': PARENT_TABLE, - 'dataType': 'string', - 'alias': PARENT_TABLE - }) + tableau_column_headers[table].append( + {"id": PARENT_ID, "dataType": "int", "alias": PARENT_ID} + ) + tableau_column_headers[table].append( + {"id": PARENT_TABLE, "dataType": "string", "alias": PARENT_TABLE} + ) for column in columns: - tableau_column_headers[table].append({ - 'id': column.get('name'), - 'dataType': column.get('type'), - 'alias': column.get('name') - }) + tableau_column_headers[table].append( + { + "id": column.get("name"), + "dataType": column.get("type"), + "alias": column.get("name"), + } + ) return tableau_column_headers def get_tableau_table_schemas(self) -> List[dict]: @@ -284,23 +294,22 @@ def get_tableau_table_schemas(self) -> List[dict]: column_headers = self.get_tableau_column_headers() for table_name, headers in column_headers.items(): table_schema = {} - table_schema['table_alias'] = table_name - table_schema['connection_name'] = f"{project}_{id_str}" - table_schema['column_headers'] = headers + table_schema["table_alias"] = table_name + table_schema["connection_name"] = f"{project}_{id_str}" + table_schema["column_headers"] = headers if table_name != DEFAULT_TABLE_NAME: - table_schema['connection_name'] += f"_{table_name}" + table_schema["connection_name"] += f"_{table_name}" ret.append(table_schema) return ret - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): self.object = self.get_object() if isinstance(self.object.content_object, XForm): self.xform = self.object.content_object - xform_json = json.loads(self.xform.json) + xform_json = self.xform.json_dict() headers = self.xform.get_headers(repeat_iterations=1) - self.flattened_dict = self.flatten_xform_columns( - xform_json.get('children')) + self.flattened_dict = self.flatten_xform_columns(xform_json.get("children")) self.xform_headers = clean_xform_headers(headers) data = self.get_tableau_table_schemas() return Response(data=data, status=status.HTTP_200_OK) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 248008cb34..f8556d63b6 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -202,7 +202,7 @@ def check_version_set(survey): """ # get the json and check for the version key - survey_json = json.loads(survey.to_json()) + survey_json = survey.to_json_dict() if not survey_json.get("version"): # set utc time as the default version survey_json["version"] = datetime.utcnow().strftime("%Y%m%d%H%M") @@ -952,19 +952,26 @@ def _set_hash(self): def _set_encrypted_field(self): if self.json and self.json != "": - json_dict = json.loads(self.json) + json_dict = self.json_dict() self.encrypted = "public_key" in json_dict def _set_public_key_field(self): if self.json and self.json != "": if self.num_of_submissions == 0 and self.public_key: - json_dict = json.loads(self.json) + json_dict = self.json_dict() json_dict["public_key"] = self.public_key survey = create_survey_element_from_dict(json_dict) - self.json = survey.to_json() + self.json = survey.to_json_dict() self.xml = survey.to_xml() self._set_encrypted_field() + def json_dict(self): + """Returns the `self.json` field data as a dict.""" + if self.json and isinstance(self.json, dict): + return self.json + + return json.loads(self.json) + def update(self, *args, **kwargs): """Persists the form to the DB.""" super().save(*args, **kwargs) @@ -1017,12 +1024,8 @@ def save(self, *args, **kwargs): # noqa: MC0001 if not self.sms_id_string and ( update_fields is None or "id_string" in update_fields ): - if isinstance(self.json, str): - self.sms_id_string = json.loads(self.json).get( - "sms_keyword", self.id_string - ) - else: - self.sms_id_string = self.json.get("sms_keyword", self.id_string) + json_dict = self.json_dict() + self.sms_id_string = json_dict.get("sms_keyword", self.id_string) if update_fields is None or "public_key" in update_fields: self._set_public_key_field() diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 4014ce18e6..990fc2f6f0 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -443,7 +443,7 @@ def set_xform_owner_data(data, xform, request, username, id_string): ) if not xform.allows_sms: data["sms_compatible"] = check_form_sms_compatibility( - None, json_survey=json.loads(xform.json) + None, json_survey=xform.json_dict() ) else: url_root = request.build_absolute_uri("/")[:-1] @@ -819,7 +819,7 @@ def edit(request, username, id_string): # noqa C901 pid = xform.sms_id_string xform.allows_sms = enabled xform.sms_id_string = sms_support_form.cleaned_data.get("sms_id_string") - compat = check_form_sms_compatibility(None, json.loads(xform.json)) + compat = check_form_sms_compatibility(None, xform.json_dict()) if compat["type"] == "alert-error": xform.allows_sms = False xform.sms_id_string = pid diff --git a/onadata/apps/sms_support/autodoc.py b/onadata/apps/sms_support/autodoc.py index d822892019..cdf0c9ce14 100644 --- a/onadata/apps/sms_support/autodoc.py +++ b/onadata/apps/sms_support/autodoc.py @@ -17,24 +17,30 @@ from builtins import str as text from onadata.apps.sms_support.tools import ( - DEFAULT_ALLOW_MEDIA, DEFAULT_DATE_FORMAT, DEFAULT_DATETIME_FORMAT, - DEFAULT_SEPARATOR, MEDIA_TYPES) + DEFAULT_ALLOW_MEDIA, + DEFAULT_DATE_FORMAT, + DEFAULT_DATETIME_FORMAT, + DEFAULT_SEPARATOR, + MEDIA_TYPES, +) def get_sample_data_for(question, json_survey, as_names=False): - """ return an example data for a particular question. + """return an example data for a particular question. - If as_names is True, returns name (not sms_field) of the question """ + If as_names is True, returns name (not sms_field) of the question""" - xlsf_name = question.get('name') - xlsf_type = question.get('type') - xlsf_choices = question.get('children') + xlsf_name = question.get("name") + xlsf_type = question.get("type") + xlsf_choices = question.get("children") now = datetime.datetime.now() - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) def safe_wrap(value): return text(value) @@ -42,169 +48,178 @@ def safe_wrap(value): if as_names: return xlsf_name - if xlsf_type == 'text': - return safe_wrap('lorem ipsum') - elif xlsf_type == 'integer': + if xlsf_type == "text": + return safe_wrap("lorem ipsum") + elif xlsf_type == "integer": return safe_wrap(4) - elif xlsf_type == 'decimal': + elif xlsf_type == "decimal": return safe_wrap(1.2) - elif xlsf_type == 'select one': - return safe_wrap( - u' '.join([c.get('sms_option') for c in xlsf_choices][:1])) - elif xlsf_type == 'select all that apply': - return safe_wrap( - u' '.join([c.get('sms_option') for c in xlsf_choices][:2])) - elif xlsf_type == 'geopoint': - return safe_wrap('12.65 -8') + elif xlsf_type == "select one": + return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:1])) + elif xlsf_type == "select all that apply": + return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:2])) + elif xlsf_type == "geopoint": + return safe_wrap("12.65 -8") elif xlsf_type in MEDIA_TYPES: - exts = {'audio': 'mp3', 'video': 'avi', 'photo': 'jpg'} - return safe_wrap('x.%s;dGhpc' % exts.get(xlsf_type, 'ext')) - elif xlsf_type == 'barcode': - return safe_wrap('abc') - elif xlsf_type == 'date': + exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} + return safe_wrap("x.%s;dGhpc" % exts.get(xlsf_type, "ext")) + elif xlsf_type == "barcode": + return safe_wrap("abc") + elif xlsf_type == "date": return safe_wrap(now.strftime(xlsf_date_fmt)) - elif xlsf_type == 'datetime': + elif xlsf_type == "datetime": return safe_wrap(now.strftime(xlsf_datetime_fmt)) - elif xlsf_type == 'note': + elif xlsf_type == "note": return None else: - return safe_wrap('?') + return safe_wrap("?") def get_helper_text(question, json_survey): - """ The full sentence (html) of the helper for a question - - Includes the type, a description - and potentialy accepted values or format """ - - xlsf_type = question.get('type') - xlsf_choices = question.get('children') - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + """The full sentence (html) of the helper for a question + + Includes the type, a description + and potentialy accepted values or format""" + + xlsf_type = question.get("type") + xlsf_choices = question.get("children") + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + ) + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR def safe_wrap(value, xlsf_type=xlsf_type): value = ( - u'%(type)s ' - u'%(text)s' % { - 'type': xlsf_type, - 'text': value - }) + '%(type)s ' + '%(text)s' + % {"type": xlsf_type, "text": value} + ) return text(value) - if xlsf_type == 'text': - return safe_wrap(u'Any string (excluding “%s”)' % separator) - elif xlsf_type == 'integer': - return safe_wrap('Any integer digit.') - elif xlsf_type == 'decimal': - return safe_wrap('A decimal or integer value.') - elif xlsf_type == 'select one': - helper = u'Select one of the following:' - helper += u'
      ' + if xlsf_type == "text": + return safe_wrap("Any string (excluding “%s”)" % separator) + elif xlsf_type == "integer": + return safe_wrap("Any integer digit.") + elif xlsf_type == "decimal": + return safe_wrap("A decimal or integer value.") + elif xlsf_type == "select one": + helper = "Select one of the following:" + helper += "
        " # pylint: disable=E1101 - helper += u''.join([ - u'
      • ' - u'%(sms_option)s %(label)s
        • ' % { - 'sms_option': c.get('sms_option'), - 'label': c.get('label') - } for c in xlsf_choices - ]) - helper += u'
      ' + helper += "".join( + [ + '
    • ' + '%(sms_option)s %(label)s
      • ' + % {"sms_option": c.get("sms_option"), "label": c.get("label")} + for c in xlsf_choices + ] + ) + helper += "
    " return safe_wrap(helper) - elif xlsf_type == 'select all that apply': - helper = u'Select none, one or more in:' - helper += u'
      ' + elif xlsf_type == "select all that apply": + helper = "Select none, one or more in:" + helper += "
        " # pylint: disable=E1101 - helper += u''.join([ - u'
      • ' - u'%(sms_option)s %(label)s
        • ' % { - 'sms_option': c.get('sms_option'), - 'label': c.get('label') - } for c in xlsf_choices - ]) - helper += u'
      ' + helper += "".join( + [ + '
    • ' + '%(sms_option)s %(label)s
      • ' + % {"sms_option": c.get("sms_option"), "label": c.get("label")} + for c in xlsf_choices + ] + ) + helper += "
    " return safe_wrap(helper) - elif xlsf_type == 'geopoint': - helper = (u'GPS coordinates as ' - u'latitude longitude.' - u'
    Optionnaly add ' - u'altitude precision after. All of them are decimal.') + elif xlsf_type == "geopoint": + helper = ( + 'GPS coordinates as ' + "latitude longitude." + '
    Optionnaly add ' + "altitude precision after. All of them are decimal." + ) return safe_wrap(helper) elif xlsf_type in MEDIA_TYPES: - exts = {'audio': 'mp3', 'video': 'avi', 'photo': 'jpg'} - helper = (u'File name and base64 data of the file as in ' - u'x.%s;dGhpc.' - u'
    It is not intented to be filled by ' - u'humans.' % exts.get(xlsf_type, 'ext')) + exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} + helper = ( + "File name and base64 data of the file as in " + 'x.%s;dGhpc.' + "
    It is not intented to be filled by " + "humans." % exts.get(xlsf_type, "ext") + ) return safe_wrap(helper) - elif xlsf_type == 'barcode': - return safe_wrap('A string representing the value behind the barcode.') - elif xlsf_type == 'date': + elif xlsf_type == "barcode": + return safe_wrap("A string representing the value behind the barcode.") + elif xlsf_type == "date": return safe_wrap( - 'A date in the format: ' - '%s' % xlsf_date_fmt) - elif xlsf_type == 'datetime': + "A date in the format: " + '%s' % xlsf_date_fmt + ) + elif xlsf_type == "datetime": return safe_wrap( - 'A datetime in the format: ' - '%s' % xlsf_datetime_fmt) + "A datetime in the format: " + '%s' % xlsf_datetime_fmt + ) else: - return safe_wrap('?') + return safe_wrap("?") def get_autodoc_for(xform): - """ The generated documentation in a dict (HTML output) + """The generated documentation in a dict (HTML output) - line_names: example line filled with question line_names - line_values: example line filled with fake (yet valid) data - helpers: list of tuples (name, text) of helper texts. + line_names: example line filled with question line_names + line_values: example line filled with fake (yet valid) data + helpers: list of tuples (name, text) of helper texts. - Helper texts are based on type of question and accepted values """ + Helper texts are based on type of question and accepted values""" - json_survey = json.loads(xform.json) + json_survey = xform.json_dict() # setup formatting values - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR sms_allow_media = bool( - json_survey.get('sms_allow_media', DEFAULT_ALLOW_MEDIA) - or DEFAULT_ALLOW_MEDIA) + json_survey.get("sms_allow_media", DEFAULT_ALLOW_MEDIA) or DEFAULT_ALLOW_MEDIA + ) helpers = [] - line_names = (u'%(keyword)s' - u'%(qid)d ' % { - 'keyword': xform.sms_id_string, - 'qid': len(helpers) - }) - line_values = (u'%(keyword)s ' % { - 'keyword': xform.sms_id_string - }) - helpers.append(('keyword', u'The keyword used to identify the form.' - u'
    Omit if using a form-aware URL.')) - - for group in json_survey.get('children', {}): - sms_field = group.get('sms_field', '') - if not sms_field or sms_field.lower() == 'meta': + line_names = ( + '%(keyword)s' + "%(qid)d " % {"keyword": xform.sms_id_string, "qid": len(helpers)} + ) + line_values = '%(keyword)s ' % { + "keyword": xform.sms_id_string + } + helpers.append( + ( + "keyword", + "The keyword used to identify the form." + "
    Omit if using a form-aware URL.", + ) + ) + + for group in json_survey.get("children", {}): + sms_field = group.get("sms_field", "") + if not sms_field or sms_field.lower() == "meta": continue - line_values += (u'' - u'%(sep)s%(sms_field)s ' % { - 'sep': separator, - 'sms_field': group.get('sms_field') - }) - line_names += (u'' - u'%(sep)s%(sms_field)s ' % { - 'sep': separator, - 'sms_field': group.get('sms_field') - }) + line_values += ( + '' + "%(sep)s%(sms_field)s " + % {"sep": separator, "sms_field": group.get("sms_field")} + ) + line_names += ( + '' + "%(sep)s%(sms_field)s " + % {"sep": separator, "sms_field": group.get("sms_field")} + ) - for question in group.get('children', {}): - type_id = question.get('type') + for question in group.get("children", {}): + type_id = question.get("type") if type_id in MEDIA_TYPES and not sms_allow_media: continue @@ -213,28 +228,20 @@ def get_autodoc_for(xform): sample = get_sample_data_for(question, json_survey) if sample is None: continue - sample_name = get_sample_data_for( - question, json_survey, as_names=True) - - line_values += (u'%(value)s ' % { - 'type': type_id, - 'value': sample - }) - line_names += (u'%(value)s' - u'%(h)s ' % { - 'type': type_id, - 'value': sample_name, - 'h': qid - }) - helpers.append((sample_name, get_helper_text( - question, json_survey))) - - line_values += u'' - - return { - 'line_values': line_values, - 'line_names': line_names, - 'helpers': helpers - } + sample_name = get_sample_data_for(question, json_survey, as_names=True) + + line_values += ( + '%(value)s ' + % {"type": type_id, "value": sample} + ) + line_names += ( + '%(value)s' + "%(h)s " % {"type": type_id, "value": sample_name, "h": qid} + ) + helpers.append((sample_name, get_helper_text(question, json_survey))) + + line_values += "" + + return {"line_values": line_values, "line_names": line_names, "helpers": helpers} diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index 4cdd923fb3..eb22a80ed7 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -9,11 +9,21 @@ from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm -from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_PARSING_ERROR,\ - SMS_SUBMISSION_REFUSED, sms_media_to_file, generate_instance,\ - DEFAULT_SEPARATOR, NA_VALUE, META_FIELDS, MEDIA_TYPES,\ - DEFAULT_DATE_FORMAT, DEFAULT_DATETIME_FORMAT, SMS_SUBMISSION_ACCEPTED,\ - is_last +from onadata.apps.sms_support.tools import ( + SMS_API_ERROR, + SMS_PARSING_ERROR, + SMS_SUBMISSION_REFUSED, + sms_media_to_file, + generate_instance, + DEFAULT_SEPARATOR, + NA_VALUE, + META_FIELDS, + MEDIA_TYPES, + DEFAULT_DATE_FORMAT, + DEFAULT_DATETIME_FORMAT, + SMS_SUBMISSION_ACCEPTED, + is_last, +) from onadata.libs.utils.logger_tools import dict2xform @@ -22,28 +32,30 @@ class SMSSyntaxError(ValueError): class SMSCastingError(ValueError): - def __init__(self, message, question=None): if question: - message = _(u"%(question)s: %(message)s") % {'question': question, - 'message': message} + message = _("%(question)s: %(message)s") % { + "question": question, + "message": message, + } super(SMSCastingError, self).__init__(message) def parse_sms_text(xform, identity, text): - json_survey = json.loads(xform.json) + json_survey = xform.json_dict() - separator = json_survey.get('sms_separator', DEFAULT_SEPARATOR) \ - or DEFAULT_SEPARATOR + separator = json_survey.get("sms_separator", DEFAULT_SEPARATOR) or DEFAULT_SEPARATOR - allow_media = bool(json_survey.get('sms_allow_media', False)) + allow_media = bool(json_survey.get("sms_allow_media", False)) - xlsf_date_fmt = json_survey.get('sms_date_format', DEFAULT_DATE_FORMAT) \ - or DEFAULT_DATE_FORMAT - xlsf_datetime_fmt = json_survey.get('sms_date_format', - DEFAULT_DATETIME_FORMAT) \ + xlsf_date_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATE_FORMAT) or DEFAULT_DATE_FORMAT + ) + xlsf_datetime_fmt = ( + json_survey.get("sms_date_format", DEFAULT_DATETIME_FORMAT) or DEFAULT_DATETIME_FORMAT + ) # extract SMS data into indexed groups of values groups = {} @@ -52,67 +64,66 @@ def parse_sms_text(xform, identity, text): groups.update({group_id: [s.strip() for s in group_text.split(None)]}) def cast_sms_value(value, question, medias=[]): - ''' Check data type of value and return cleaned version ''' + """Check data type of value and return cleaned version""" - xlsf_type = question.get('type') - xlsf_name = question.get('name') - xlsf_choices = question.get('children') - xlsf_required = bool(question.get('bind', {}) - .get('required', '').lower() in ('yes', 'true')) + xlsf_type = question.get("type") + xlsf_name = question.get("name") + xlsf_choices = question.get("children") + xlsf_required = bool( + question.get("bind", {}).get("required", "").lower() in ("yes", "true") + ) # we don't handle constraint for now as it's a little complex and # unsafe. # xlsf_constraint=question.get('constraint') if xlsf_required and not len(value): - raise SMSCastingError(_(u"Required field missing"), xlsf_name) + raise SMSCastingError(_("Required field missing"), xlsf_name) def safe_wrap(func): try: return func() except Exception as e: - raise SMSCastingError(_(u"%(error)s") % {'error': e}, - xlsf_name) + raise SMSCastingError(_("%(error)s") % {"error": e}, xlsf_name) def media_value(value, medias): - ''' handle media values + """handle media values - extract name and base64 data. - fills the media holder with (name, data) tuple ''' + extract name and base64 data. + fills the media holder with (name, data) tuple""" try: - filename, b64content = value.split(';', 1) - medias.append((filename, - base64.b64decode(b64content))) + filename, b64content = value.split(";", 1) + medias.append((filename, base64.b64decode(b64content))) return filename except Exception as e: - raise SMSCastingError(_(u"Media file format " - u"incorrect. %(except)r") - % {'except': e}, xlsf_name) + raise SMSCastingError( + _("Media file format " "incorrect. %(except)r") % {"except": e}, + xlsf_name, + ) - if xlsf_type == 'text': + if xlsf_type == "text": return safe_wrap(lambda: str(value)) - elif xlsf_type == 'integer': + elif xlsf_type == "integer": return safe_wrap(lambda: int(value)) - elif xlsf_type == 'decimal': + elif xlsf_type == "decimal": return safe_wrap(lambda: float(value)) - elif xlsf_type == 'select one': + elif xlsf_type == "select one": for choice in xlsf_choices: - if choice.get('sms_option') == value: - return choice.get('name') - raise SMSCastingError(_(u"No matching choice " - u"for '%(input)s'") - % {'input': value}, - xlsf_name) - elif xlsf_type == 'select all that apply': + if choice.get("sms_option") == value: + return choice.get("name") + raise SMSCastingError( + _("No matching choice " "for '%(input)s'") % {"input": value}, xlsf_name + ) + elif xlsf_type == "select all that apply": values = [s.strip() for s in value.split()] ret_values = [] for indiv_value in values: for choice in xlsf_choices: - if choice.get('sms_option') == indiv_value: - ret_values.append(choice.get('name')) - return u" ".join(ret_values) - elif xlsf_type == 'geopoint': - err_msg = _(u"Incorrect geopoint coordinates.") + if choice.get("sms_option") == indiv_value: + ret_values.append(choice.get("name")) + return " ".join(ret_values) + elif xlsf_type == "geopoint": + err_msg = _("Incorrect geopoint coordinates.") geodata = [s.strip() for s in value.split()] if len(geodata) < 2 and len(geodata) > 4: raise SMSCastingError(err_msg, xlsf_name) @@ -137,28 +148,27 @@ def media_value(value, medias): # file_name;base64 encodeed content. # Example: hello.jpg;dGhpcyBpcyBteSBwaWN0dXJlIQ== return media_value(value, medias) - elif xlsf_type == 'barcode': + elif xlsf_type == "barcode": return safe_wrap(lambda: text(value)) - elif xlsf_type == 'date': - return safe_wrap(lambda: datetime.strptime(value, - xlsf_date_fmt).date()) - elif xlsf_type == 'datetime': - return safe_wrap(lambda: datetime.strptime(value, - xlsf_datetime_fmt)) - elif xlsf_type == 'note': - return safe_wrap(lambda: '') - raise SMSCastingError(_(u"Unsuported column '%(type)s'") - % {'type': xlsf_type}, xlsf_name) + elif xlsf_type == "date": + return safe_wrap(lambda: datetime.strptime(value, xlsf_date_fmt).date()) + elif xlsf_type == "datetime": + return safe_wrap(lambda: datetime.strptime(value, xlsf_datetime_fmt)) + elif xlsf_type == "note": + return safe_wrap(lambda: "") + raise SMSCastingError( + _("Unsuported column '%(type)s'") % {"type": xlsf_type}, xlsf_name + ) def get_meta_value(xlsf_type, identity): - ''' XLSForm Meta field value ''' - if xlsf_type in ('deviceid', 'subscriberid', 'imei'): + """XLSForm Meta field value""" + if xlsf_type in ("deviceid", "subscriberid", "imei"): return NA_VALUE - elif xlsf_type in ('start', 'end'): + elif xlsf_type in ("start", "end"): return datetime.now().isoformat() - elif xlsf_type == 'today': + elif xlsf_type == "today": return date.today().isoformat() - elif xlsf_type == 'phonenumber': + elif xlsf_type == "phonenumber": return identity return NA_VALUE @@ -170,24 +180,24 @@ def get_meta_value(xlsf_type, identity): notes = [] # loop on all XLSForm questions - for expected_group in json_survey.get('children', [{}]): - if not expected_group.get('type') == 'group': + for expected_group in json_survey.get("children", [{}]): + if not expected_group.get("type") == "group": # non-grouped questions are not valid for SMS continue # retrieve part of SMS text for this group - group_id = expected_group.get('sms_field') + group_id = expected_group.get("sms_field") answers = groups.get(group_id) - if not group_id or (not answers and not group_id.startswith('meta')): + if not group_id or (not answers and not group_id.startswith("meta")): # group is not meant to be filled by SMS # or hasn't been filled continue # Add a holder for this group's answers data - survey_answers.update({expected_group.get('name'): {}}) + survey_answers.update({expected_group.get("name"): {}}) # retrieve question definition for each answer - egroups = expected_group.get('children', [{}]) + egroups = expected_group.get("children", [{}]) # number of intermediate, omited questions (medias) step_back = 0 @@ -195,15 +205,15 @@ def get_meta_value(xlsf_type, identity): real_value = None - question_type = question.get('type') - if question_type in ('calculate'): + question_type = question.get("type") + if question_type in ("calculate"): # 'calculate' question are not implemented. # 'note' ones are just meant to be displayed on device continue - if question_type == 'note': - if not question.get('constraint', ''): - notes.append(question.get('label')) + if question_type == "note": + if not question.get("constraint", ""): + notes.append(question.get("label")) continue if not allow_media and question_type in MEDIA_TYPES: @@ -219,40 +229,41 @@ def get_meta_value(xlsf_type, identity): if question_type in META_FIELDS: # some question are not to be fed by users - real_value = get_meta_value(xlsf_type=question_type, - identity=identity) + real_value = get_meta_value(xlsf_type=question_type, identity=identity) else: # actual SMS-sent answer. # Only last answer/question of each group is allowed # to have multiple spaces if is_last(idx, egroups): - answer = u" ".join(answers[idx:]) + answer = " ".join(answers[idx:]) else: answer = answers[sidx] if real_value is None: # retrieve actual value and fail if it doesn't meet reqs. - real_value = cast_sms_value(answer, - question=question, medias=medias) + real_value = cast_sms_value(answer, question=question, medias=medias) # set value to its question name - survey_answers[expected_group.get('name')] \ - .update({question.get('name'): real_value}) + survey_answers[expected_group.get("name")].update( + {question.get("name"): real_value} + ) return survey_answers, medias, notes -def process_incoming_smses(username, incomings, - id_string=None): - ''' Process Incoming (identity, text[, id_string]) SMS ''' +def process_incoming_smses(username, incomings, id_string=None): + """Process Incoming (identity, text[, id_string]) SMS""" xforms = [] medias = [] xforms_notes = [] responses = [] json_submissions = [] - resp_str = {'success': _(u"[SUCCESS] Your submission has been accepted. " - u"It's ID is {{ id }}.")} + resp_str = { + "success": _( + "[SUCCESS] Your submission has been accepted. " "It's ID is {{ id }}." + ) + } def process_incoming(incoming, id_string): # assign variables @@ -263,72 +274,90 @@ def process_incoming(incoming, id_string): if id_string is None and len(incoming) >= 3: id_string = incoming[2] else: - responses.append({'code': SMS_API_ERROR, - 'text': _(u"Missing 'identity' " - u"or 'text' field.")}) + responses.append( + { + "code": SMS_API_ERROR, + "text": _("Missing 'identity' " "or 'text' field."), + } + ) return if not len(identity.strip()) or not len(text.strip()): - responses.append({'code': SMS_API_ERROR, - 'text': _(u"'identity' and 'text' fields can " - u"not be empty.")}) + responses.append( + { + "code": SMS_API_ERROR, + "text": _("'identity' and 'text' fields can " "not be empty."), + } + ) return # if no id_string has been supplied # we expect the SMS to be prefixed with the form's sms_id_string if id_string is None: keyword, text = [s.strip() for s in text.split(None, 1)] - xform = XForm.objects.get(user__username=username, - sms_id_string=keyword) + xform = XForm.objects.get(user__username=username, sms_id_string=keyword) else: - xform = XForm.objects.get(user__username=username, - id_string=id_string) + xform = XForm.objects.get(user__username=username, id_string=id_string) if not xform.allows_sms: - responses.append({'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"The form '%(id_string)s' does not " - u"accept SMS submissions.") - % {'id_string': xform.id_string}}) + responses.append( + { + "code": SMS_SUBMISSION_REFUSED, + "text": _( + "The form '%(id_string)s' does not " "accept SMS submissions." + ) + % {"id_string": xform.id_string}, + } + ) return # parse text into a dict object of groups with values - json_submission, medias_submission, notes = parse_sms_text(xform, - identity, - text) + json_submission, medias_submission, notes = parse_sms_text( + xform, identity, text + ) # retrieve sms_response if exist in the form. - json_survey = json.loads(xform.json) - if json_survey.get('sms_response'): - resp_str.update({'success': json_survey.get('sms_response')}) + json_survey = xform.json_dict() + if json_survey.get("sms_response"): + resp_str.update({"success": json_survey.get("sms_response")}) # check that the form contains at least one filled group - meta_groups = sum([1 for k in list(json_submission) - if k.startswith('meta')]) + meta_groups = sum([1 for k in list(json_submission) if k.startswith("meta")]) if len(list(json_submission)) <= meta_groups: - responses.append({'code': SMS_PARSING_ERROR, - 'text': _(u"There must be at least one group of " - u"questions filled.")}) + responses.append( + { + "code": SMS_PARSING_ERROR, + "text": _( + "There must be at least one group of " "questions filled." + ), + } + ) return # check that required fields have been filled - required_fields = [f.get('name') - for g in json_survey.get('children', {}) - for f in g.get('children', {}) - if f.get('bind', {}).get('required', 'no') == 'yes'] + required_fields = [ + f.get("name") + for g in json_survey.get("children", {}) + for f in g.get("children", {}) + if f.get("bind", {}).get("required", "no") == "yes" + ] submitted_fields = {} for group in json_submission.values(): submitted_fields.update(group) for field in required_fields: if not submitted_fields.get(field): - responses.append({'code': SMS_SUBMISSION_REFUSED, - 'text': _(u"Required field `%(field)s` is " - u"missing.") % {'field': field}}) + responses.append( + { + "code": SMS_SUBMISSION_REFUSED, + "text": _("Required field `%(field)s` is " "missing.") + % {"field": field}, + } + ) return # convert dict object into an XForm string - xml_submission = dict2xform(jsform=json_submission, - form_id=xform.id_string) + xml_submission = dict2xform(jsform=json_submission, form_id=xform.id_string) # compute notes data = {} @@ -336,13 +365,12 @@ def process_incoming(incoming, id_string): data.update(g) for idx, note in enumerate(notes): try: - notes[idx] = note.replace('${', '{').format(**data) + notes[idx] = note.replace("${", "{").format(**data) except Exception as e: - logging.exception(_(u'Updating note threw exception: %s' - % text(e))) + logging.exception(_("Updating note threw exception: %s" % text(e))) # process_incoming expectes submission to be a file-like object - xforms.append(BytesIO(xml_submission.encode('utf-8'))) + xforms.append(BytesIO(xml_submission.encode("utf-8"))) medias.append(medias_submission) json_submissions.append(json_submission) xforms_notes.append(notes) @@ -351,29 +379,31 @@ def process_incoming(incoming, id_string): try: process_incoming(incoming, id_string) except Exception as e: - responses.append({'code': SMS_PARSING_ERROR, 'text': text(e)}) + responses.append({"code": SMS_PARSING_ERROR, "text": text(e)}) for idx, xform in enumerate(xforms): # generate_instance expects media as a request.FILES.values() list xform_medias = [sms_media_to_file(f, n) for n, f in medias[idx]] # create the instance in the data base - response = generate_instance(username=username, - xml_file=xform, - media_files=xform_medias) - if response.get('code') == SMS_SUBMISSION_ACCEPTED: - success_response = re.sub(r'{{\s*[i,d,I,D]{2}\s*}}', - response.get('id'), - resp_str.get('success'), re.I) + response = generate_instance( + username=username, xml_file=xform, media_files=xform_medias + ) + if response.get("code") == SMS_SUBMISSION_ACCEPTED: + success_response = re.sub( + r"{{\s*[i,d,I,D]{2}\s*}}", + response.get("id"), + resp_str.get("success"), + re.I, + ) # extend success_response with data from the answers data = {} for g in json_submissions[idx].values(): data.update(g) - success_response = success_response.replace('${', - '{').format(**data) - response.update({'text': success_response}) + success_response = success_response.replace("${", "{").format(**data) + response.update({"text": success_response}) # add sendouts (notes) - response.update({'sendouts': xforms_notes[idx]}) + response.update({"sendouts": xforms_notes[idx]}) responses.append(response) return responses diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 8a315507f5..d73de5c7c4 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -187,7 +187,7 @@ def save(self, *args, **kwargs): survey["name"] = default_name else: survey["id_string"] = self.id_string - self.json = survey.to_json() + self.json = survey.to_json_dict() self.xml = survey.to_xml() self.version = survey.get("version") self.last_updated_at = timezone.now() diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 3882904d35..7984695898 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -438,7 +438,7 @@ def _get_name_for_type(self, type_value): representation what the 'name' was for a given type_value ('start' or 'end') """ - datadict = json.loads(self.instance.xform.json) + datadict = self.instance.xform.json_dict() for item in datadict["children"]: if isinstance(item, dict) and item.get("type") == type_value: return item["name"] diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index a1c4b6d6a9..a2423eb601 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -121,6 +121,6 @@ def get_field_xpath(self, obj): return None question_name = dict_key_for_value(qa_dict, obj.filename) - data = json.loads(obj.instance.xform.json) + data = obj.instance.xform.json_dict() return get_path(data, question_name, []) diff --git a/onadata/libs/serializers/merged_xform_serializer.py b/onadata/libs/serializers/merged_xform_serializer.py index c52c57509f..556d3af8c7 100644 --- a/onadata/libs/serializers/merged_xform_serializer.py +++ b/onadata/libs/serializers/merged_xform_serializer.py @@ -22,24 +22,28 @@ def _get_fields_set(xform): - return [(element.get_abbreviated_xpath(), element.type) - for element in xform.survey_elements - if element.type not in ['', 'survey']] + return [ + (element.get_abbreviated_xpath(), element.type) + for element in xform.survey_elements + if element.type not in ["", "survey"] + ] def _get_elements(elements, intersect, parent_prefix=None): new_elements = [] for element in elements: - name = element['name'] - name = name if not parent_prefix else '/'.join([parent_prefix, name]) + name = element["name"] + name = name if not parent_prefix else "/".join([parent_prefix, name]) if name in intersect: k = element.copy() - if 'children' in element and element['type'] not in SELECTS: - k['children'] = _get_elements( - element['children'], - [__ for __ in intersect if __.startswith(name)], name) - if not k['children']: + if "children" in element and element["type"] not in SELECTS: + k["children"] = _get_elements( + element["children"], + [__ for __ in intersect if __.startswith(name)], + name, + ) + if not k["children"]: continue new_elements.append(k) @@ -54,44 +58,44 @@ def get_merged_xform_survey(xforms): :param xforms: A list of XForms of at least length 2. """ if len(xforms) < 2: - raise serializers.ValidationError(_('Expecting at least 2 xforms')) + raise serializers.ValidationError(_("Expecting at least 2 xforms")) xform_sets = [_get_fields_set(xform) for xform in xforms] - merged_xform_dict = json.loads(xforms[0].json) - children = merged_xform_dict.pop('children') - merged_xform_dict['children'] = [] + merged_xform_dict = xforms[0].json_dict() + children = merged_xform_dict.pop("children") + merged_xform_dict["children"] = [] intersect = set(xform_sets[0]).intersection(*xform_sets[1:]) intersect = set([__ for (__, ___) in intersect]) - merged_xform_dict['children'] = _get_elements(children, intersect) + merged_xform_dict["children"] = _get_elements(children, intersect) - if '_xpath' in merged_xform_dict: - del merged_xform_dict['_xpath'] + if "_xpath" in merged_xform_dict: + del merged_xform_dict["_xpath"] is_empty = True - xform_dicts = [json.loads(xform.json) for xform in xforms] - for child in merged_xform_dict['children']: - if child['name'] != 'meta' and is_empty: + xform_dicts = [xform.json_dict() for xform in xforms] + for child in merged_xform_dict["children"]: + if child["name"] != "meta" and is_empty: is_empty = False # Remove bind attributes from child elements - if 'bind' in child: - del child['bind'] + if "bind" in child: + del child["bind"] # merge select one and select multiple options - if 'children' in child and child['type'] in SELECTS: + if "children" in child and child["type"] in SELECTS: children = [] for xform_dict in xform_dicts: - element_name = child['name'] + element_name = child["name"] element_list = list( - filter(lambda x: x['name'] == element_name, - xform_dict['children'])) + filter(lambda x: x["name"] == element_name, xform_dict["children"]) + ) if element_list and element_list[0]: - children += element_list[0]['children'] + children += element_list[0]["children"] # remove duplicates set_of_jsons = {json.dumps(d, sort_keys=True) for d in children} - child['children'] = [json.loads(t) for t in set_of_jsons] + child["children"] = [json.loads(t) for t in set_of_jsons] if is_empty: raise serializers.ValidationError(_("No matching fields in xforms.")) @@ -103,11 +107,11 @@ def minimum_two_xforms(value): """Validate at least 2 xforms are provided""" if len(value) < 2: raise serializers.ValidationError( - _('This field should have at least two unique xforms.')) + _("This field should have at least two unique xforms.") + ) if len(set(value)) != len(value): - raise serializers.ValidationError( - _('This field should have unique xforms')) + raise serializers.ValidationError(_("This field should have unique xforms")) return value @@ -125,14 +129,23 @@ class XFormSerializer(serializers.HyperlinkedModelSerializer): """XFormSerializer""" url = serializers.HyperlinkedIdentityField( - view_name='xform-detail', lookup_field='pk') - owner = serializers.CharField(source='user.username') - project_name = serializers.CharField(source='project.name') + view_name="xform-detail", lookup_field="pk" + ) + owner = serializers.CharField(source="user.username") + project_name = serializers.CharField(source="project.name") class Meta: model = XForm - fields = ('url', 'id', 'id_string', 'title', 'num_of_submissions', - 'owner', 'project_id', 'project_name') + fields = ( + "url", + "id", + "id_string", + "title", + "num_of_submissions", + "owner", + "project_id", + "project_name", + ) class XFormListField(serializers.ManyRelatedField): @@ -140,33 +153,46 @@ class XFormListField(serializers.ManyRelatedField): def to_representation(self, iterable): return [ - dict(i) for i in XFormSerializer( - iterable, many=True, context=self.context).data + dict(i) + for i in XFormSerializer(iterable, many=True, context=self.context).data ] class MergedXFormSerializer(serializers.HyperlinkedModelSerializer): """MergedXForm Serializer to create and update merged datasets""" + url = serializers.HyperlinkedIdentityField( - view_name='merged-xform-detail', lookup_field='pk') - name = serializers.CharField( - max_length=XFORM_TITLE_LENGTH, write_only=True) + view_name="merged-xform-detail", lookup_field="pk" + ) + name = serializers.CharField(max_length=XFORM_TITLE_LENGTH, write_only=True) xforms = XFormListField( allow_empty=False, child_relation=serializers.HyperlinkedRelatedField( allow_empty=False, queryset=XForm.objects.filter( - is_merged_dataset=False, deleted_at__isnull=True), - view_name='xform-detail'), - validators=[minimum_two_xforms, has_matching_fields]) + is_merged_dataset=False, deleted_at__isnull=True + ), + view_name="xform-detail", + ), + validators=[minimum_two_xforms, has_matching_fields], + ) num_of_submissions = serializers.SerializerMethodField() last_submission_time = serializers.SerializerMethodField() class Meta: model = MergedXForm - fields = ('url', 'id', 'xforms', 'name', 'project', 'title', - 'num_of_submissions', 'last_submission_time', 'uuid') - write_only_fields = ('uuid', ) + fields = ( + "url", + "id", + "xforms", + "name", + "project", + "title", + "num_of_submissions", + "last_submission_time", + "uuid", + ) + write_only_fields = ("uuid",) # pylint: disable=no-self-use def get_num_of_submissions(self, obj): @@ -175,7 +201,7 @@ def get_num_of_submissions(self, obj): 'num_of_submissions'. """ - value = getattr(obj, 'number_of_submissions', obj.num_of_submissions) + value = getattr(obj, "number_of_submissions", obj.num_of_submissions) return value @@ -183,41 +209,41 @@ def get_last_submission_time(self, obj): """Return datetime of last submission from all forms""" values = [ x.last_submission_time.isoformat() - for x in obj.xforms.only('last_submission_time') + for x in obj.xforms.only("last_submission_time") if x.last_submission_time ] if values: return sorted(values, reverse=True)[0] def create(self, validated_data): - request = self.context['request'] - xforms = validated_data['xforms'] + request = self.context["request"] + xforms = validated_data["xforms"] # create merged xml, json with non conflicting id_string survey = get_merged_xform_survey(xforms) - survey['id_string'] = base64.b64encode( - uuid.uuid4().hex[:6].encode('utf-8')).decode('utf-8') - survey['sms_keyword'] = survey['id_string'] - survey['title'] = validated_data.pop('name') - validated_data['json'] = survey.to_json() + survey["id_string"] = base64.b64encode( + uuid.uuid4().hex[:6].encode("utf-8") + ).decode("utf-8") + survey["sms_keyword"] = survey["id_string"] + survey["title"] = validated_data.pop("name") + validated_data["json"] = survey.to_json() try: - validated_data['xml'] = survey.to_xml() + validated_data["xml"] = survey.to_xml() except PyXFormError as error: - raise serializers.ValidationError({ - 'xforms': - _("Problem Merging the Form: {}".format(error)) - }) - validated_data['user'] = validated_data['project'].user - validated_data['created_by'] = request.user - validated_data['is_merged_dataset'] = True - validated_data['num_of_submissions'] = sum( - [__.num_of_submissions for __ in validated_data.get('xforms')]) - validated_data['instances_with_geopoints'] = any([ - __.instances_with_geopoints for __ in validated_data.get('xforms') - ]) + raise serializers.ValidationError( + {"xforms": _("Problem Merging the Form: {}".format(error))} + ) + validated_data["user"] = validated_data["project"].user + validated_data["created_by"] = request.user + validated_data["is_merged_dataset"] = True + validated_data["num_of_submissions"] = sum( + [__.num_of_submissions for __ in validated_data.get("xforms")] + ) + validated_data["instances_with_geopoints"] = any( + [__.instances_with_geopoints for __ in validated_data.get("xforms")] + ) with transaction.atomic(): - instance = super(MergedXFormSerializer, - self).create(validated_data) + instance = super(MergedXFormSerializer, self).create(validated_data) if instance.xforms.all().count() == 0 and xforms: for xform in xforms: diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index fa605aa999..948ddb4aa3 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -3,7 +3,6 @@ CSV data import module. """ import functools -import json import logging import sys import uuid @@ -101,7 +100,7 @@ def dict2xmlsubmission(submission_dict, xform, instance_id, submission_date): :rtype: string """ - xform_dict = xform.json if isinstance(xform.json, dict) else json.loads(xform.json) + xform_dict = xform.json_dict() return ( '' '<{0} id="{1}" instanceID="uuid:{2}" submissionDate="{3}">{4}' @@ -324,7 +323,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): csv_file.seek(0) csv_reader = ucsv.DictReader(csv_file, encoding="utf-8-sig") - xform_json = xform.json if isinstance(xform.json, dict) else json.loads(xform.json) + xform_json = xform.json_dict() select_multiples = [ qstn.name for qstn in xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE) ] From 1dbbecac45c3e4af81809dfa76e74462d6cc6846 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 13:42:20 +0300 Subject: [PATCH 198/234] Check for None value in textit rest service configuration --- onadata/libs/models/textit_service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/onadata/libs/models/textit_service.py b/onadata/libs/models/textit_service.py index 10e03264b0..a5bc4492ed 100644 --- a/onadata/libs/models/textit_service.py +++ b/onadata/libs/models/textit_service.py @@ -91,6 +91,13 @@ def retrieve(self): """ data_value = MetaData.textit(self.xform) + if data_value is None: + raise serializers.ValidationError( + _( + "Error occurred when loading textit service." + "Resolve by updating auth_token, flow_uuid and contacts fields" + ) + ) try: self.auth_token, self.flow_uuid, self.contacts = data_value.split( METADATA_SEPARATOR From 9c062679e67f596d2be311f1406576c7a9a7dce8 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 14:14:19 +0300 Subject: [PATCH 199/234] fixes: Check if xform.json is a dict --- .../api/tests/viewsets/test_xform_viewset.py | 4 +- onadata/apps/logger/models/xform.py | 2 +- onadata/apps/sms_support/autodoc.py | 1 - onadata/apps/sms_support/parser.py | 1 - .../libs/serializers/attachment_serializer.py | 1 - onadata/libs/serializers/floip_serializer.py | 152 ++--- onadata/libs/tests/utils/test_csv_import.py | 432 ++++++++------- onadata/libs/utils/api_export_tools.py | 5 +- onadata/libs/utils/logger_tools.py | 524 ++++++++++-------- 9 files changed, 621 insertions(+), 501 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 5aad3c0c17..032505f875 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -3853,7 +3853,7 @@ def test_versions_endpoint(self): returned_data.pop("date_modified") self.assertEqual(returned_data, expected_data) old_version = self.xform.version - expected_json = self.xform.json + expected_json = self.xform.json_dict() expected_xml = self.xform.xml # Replace form @@ -3879,7 +3879,7 @@ def test_versions_endpoint(self): ) response = view(request, pk=self.xform.pk, version_id=old_version) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data, json.loads(expected_json)) + self.assertEqual(response.data, expected_json) response = view(request, pk=self.xform.pk, version_id=old_version, format="xml") self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index f8556d63b6..0f43ebba1e 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -967,7 +967,7 @@ def _set_public_key_field(self): def json_dict(self): """Returns the `self.json` field data as a dict.""" - if self.json and isinstance(self.json, dict): + if isinstance(self.json, dict): return self.json return json.loads(self.json) diff --git a/onadata/apps/sms_support/autodoc.py b/onadata/apps/sms_support/autodoc.py index cdf0c9ce14..4084140cb8 100644 --- a/onadata/apps/sms_support/autodoc.py +++ b/onadata/apps/sms_support/autodoc.py @@ -13,7 +13,6 @@ from __future__ import absolute_import import datetime -import json from builtins import str as text from onadata.apps.sms_support.tools import ( diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index eb22a80ed7..b28d40a870 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -1,5 +1,4 @@ import base64 -import json import logging import re from builtins import str as text diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index a2423eb601..7c676b2fc4 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -2,7 +2,6 @@ """ Attachments serializer. """ -import json from six import itervalues diff --git a/onadata/libs/serializers/floip_serializer.py b/onadata/libs/serializers/floip_serializer.py index 07b05df0b0..6dc9994b59 100644 --- a/onadata/libs/serializers/floip_serializer.py +++ b/onadata/libs/serializers/floip_serializer.py @@ -26,10 +26,10 @@ from onadata.apps.logger.models import XForm from onadata.libs.utils.logger_tools import dict2xform, safe_create_instance -CONTACT_ID_INDEX = getattr(settings, 'FLOW_RESULTS_CONTACT_ID_INDEX', 2) -SESSION_ID_INDEX = getattr(settings, 'FLOW_RESULTS_SESSION_ID_INDEX', 3) -QUESTION_INDEX = getattr(settings, 'FLOW_RESULTS_QUESTION_INDEX', 4) -ANSWER_INDEX = getattr(settings, 'FLOW_RESULTS_ANSWER_INDEX', 5) +CONTACT_ID_INDEX = getattr(settings, "FLOW_RESULTS_CONTACT_ID_INDEX", 2) +SESSION_ID_INDEX = getattr(settings, "FLOW_RESULTS_SESSION_ID_INDEX", 3) +QUESTION_INDEX = getattr(settings, "FLOW_RESULTS_QUESTION_INDEX", 4) +ANSWER_INDEX = getattr(settings, "FLOW_RESULTS_ANSWER_INDEX", 5) def _get_user(username): @@ -39,21 +39,24 @@ def _get_user(username): def _get_owner(request): - owner = request.data.get('owner') or request.user + owner = request.data.get("owner") or request.user if isinstance(owner, six.string_types): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError( - _(u"User with username %s does not exist." % owner)) + raise ValidationError(_("User with username %s does not exist." % owner)) return owner_obj return owner -def parse_responses(responses, session_id_index=SESSION_ID_INDEX, - question_index=QUESTION_INDEX, answer_index=ANSWER_INDEX, - contact_id_index=CONTACT_ID_INDEX): +def parse_responses( + responses, + session_id_index=SESSION_ID_INDEX, + question_index=QUESTION_INDEX, + answer_index=ANSWER_INDEX, + contact_id_index=CONTACT_ID_INDEX, +): """ Returns individual submission for all responses in a flow-results responses package. @@ -65,11 +68,11 @@ def parse_responses(responses, session_id_index=SESSION_ID_INDEX, continue if current_key is None: current_key = row[session_id_index] - if 'meta' not in submission: - submission['meta'] = { - 'instanceID': 'uuid:%s' % current_key, - 'sessionID': current_key, - 'contactID': row[contact_id_index] + if "meta" not in submission: + submission["meta"] = { + "instanceID": "uuid:%s" % current_key, + "sessionID": current_key, + "contactID": row[contact_id_index], } if current_key != row[session_id_index]: yield submission @@ -84,6 +87,7 @@ class ReadOnlyUUIDField(serializers.ReadOnlyField): """ Custom ReadOnlyField for UUID """ + def to_representation(self, obj): # pylint: disable=no-self-use return str(UUID(obj)) @@ -93,33 +97,38 @@ class FloipListSerializer(serializers.HyperlinkedModelSerializer): """ FloipListSerializer class. """ + url = serializers.HyperlinkedIdentityField( - view_name='flow-results-detail', lookup_field='uuid') - id = ReadOnlyUUIDField(source='uuid') # pylint: disable=C0103 - name = serializers.ReadOnlyField(source='id_string') - created = serializers.ReadOnlyField(source='date_created') - modified = serializers.ReadOnlyField(source='date_modified') + view_name="flow-results-detail", lookup_field="uuid" + ) + id = ReadOnlyUUIDField(source="uuid") # pylint: disable=C0103 + name = serializers.ReadOnlyField(source="id_string") + created = serializers.ReadOnlyField(source="date_created") + modified = serializers.ReadOnlyField(source="date_modified") class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'packages' + + resource_name = "packages" class Meta: model = XForm - fields = ('url', 'id', 'name', 'title', 'created', 'modified') + fields = ("url", "id", "name", "title", "created", "modified") class FloipSerializer(serializers.HyperlinkedModelSerializer): """ FloipSerializer class. """ + url = serializers.HyperlinkedIdentityField( - view_name='floip-detail', lookup_field='pk') + view_name="floip-detail", lookup_field="pk" + ) profile = serializers.SerializerMethodField() - created = serializers.ReadOnlyField(source='date_created') - modified = serializers.ReadOnlyField(source='date_modified') + created = serializers.ReadOnlyField(source="date_created") + modified = serializers.ReadOnlyField(source="date_modified") # pylint: disable=invalid-name flow_results_specification_version = serializers.SerializerMethodField() resources = serializers.SerializerMethodField() @@ -128,26 +137,35 @@ class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'packages' + + resource_name = "packages" class Meta: model = XForm - fields = ('url', 'id', 'id_string', 'title', 'profile', 'created', - 'modified', 'flow_results_specification_version', - 'resources') + fields = ( + "url", + "id", + "id_string", + "title", + "profile", + "created", + "modified", + "flow_results_specification_version", + "resources", + ) def get_profile(self, value): # pylint: disable=no-self-use,W0613 """ Returns the data-package profile. """ - return 'data-package' + return "data-package" # pylint: disable=no-self-use,unused-argument def get_flow_results_specification_version(self, value): """ Returns the flow results specification version. """ - return '1.0.0-rc1' + return "1.0.0-rc1" def get_resources(self, value): # pylint: disable=no-self-use,W0613 """ @@ -158,26 +176,27 @@ def get_resources(self, value): # pylint: disable=no-self-use,W0613 def _process_request(self, request, update_instance=None): data = deepcopy(request.data) - if 'profile' in data and data['profile'] == 'flow-results-package': - data['profile'] = 'data-package' - descriptor = BytesIO(json.dumps(data).encode('utf-8')) + if "profile" in data and data["profile"] == "flow-results-package": + data["profile"] = "data-package" + descriptor = BytesIO(json.dumps(data).encode("utf-8")) descriptor.seek(0, os.SEEK_END) floip_file = InMemoryUploadedFile( descriptor, - 'floip_file', - request.data.get('name') + '.json', - 'application/json', + "floip_file", + request.data.get("name") + ".json", + "application/json", descriptor.tell(), - charset=None) + charset=None, + ) kwargs = { - 'user': request.user, - 'post': None, - 'files': {'floip_file': floip_file}, - 'owner': request.user, + "user": request.user, + "post": None, + "files": {"floip_file": floip_file}, + "owner": request.user, } if update_instance: - kwargs['id_string'] = update_instance.id_string - kwargs['project'] = update_instance.project + kwargs["id_string"] = update_instance.id_string + kwargs["project"] = update_instance.project instance = do_publish_xlsform(**kwargs) if isinstance(instance, XForm): return instance @@ -185,27 +204,32 @@ def _process_request(self, request, update_instance=None): raise serializers.ValidationError(instance) def create(self, validated_data): - request = self.context['request'] + request = self.context["request"] return self._process_request(request) def update(self, instance, validated_data): - request = self.context['request'] + request = self.context["request"] return self._process_request(request, instance) def to_representation(self, instance): - request = self.context['request'] + request = self.context["request"] data_id = str(UUID(instance.uuid)) data_url = request.build_absolute_uri( - reverse('flow-results-responses', kwargs={'uuid': data_id})) + reverse("flow-results-responses", kwargs={"uuid": data_id}) + ) package = survey_to_floip_package( - json.loads(instance.json), data_id, instance.date_created, - instance.date_modified, data_url) + instance.json_dict(), + data_id, + instance.date_created, + instance.date_modified, + data_url, + ) data = package.descriptor - if data['profile'] != 'flow-results-package': - data['profile'] = 'flow-results-package' + if data["profile"] != "flow-results-package": + data["profile"] = "flow-results-package" return data @@ -214,6 +238,7 @@ class FlowResultsResponse(object): # pylint: disable=too-few-public-methods """ FLowResultsResponse class to hold a list of submission ids. """ + id = None # pylint: disable=invalid-name responses = [] duplicates = 0 @@ -229,6 +254,7 @@ class FlowResultsResponseSerializer(serializers.Serializer): FlowResultsResponseSerializer for handling publishing of Flow Results Response package. """ + id = serializers.CharField() # pylint: disable=invalid-name responses = serializers.ListField() duplicates = serializers.IntegerField(read_only=True) @@ -237,23 +263,25 @@ class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 """ JSON API metaclass. """ - resource_name = 'responses' + + resource_name = "responses" def create(self, validated_data): duplicates = 0 - request = self.context['request'] - responses = validated_data['responses'] - uuid = UUID(validated_data['id']) + request = self.context["request"] + responses = validated_data["responses"] + uuid = UUID(validated_data["id"]) xform = get_object_or_404( - XForm, - Q(uuid=str(uuid)) | Q(uuid=uuid.hex), - deleted_at__isnull=True) + XForm, Q(uuid=str(uuid)) | Q(uuid=uuid.hex), deleted_at__isnull=True + ) for submission in parse_responses(responses): - xml_file = BytesIO(dict2xform( - submission, xform.id_string, 'data').encode('utf-8')) + xml_file = BytesIO( + dict2xform(submission, xform.id_string, "data").encode("utf-8") + ) error, _instance = safe_create_instance( - request.user.username, xml_file, [], None, request) + request.user.username, xml_file, [], None, request + ) if error and error.status_code != 202: raise serializers.ValidationError(error) if error and error.status_code == 202: diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index 0991a6eea5..fb55f8ce39 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import json import os import re from builtins import open @@ -16,8 +15,11 @@ from onadata.apps.logger.models import Instance, XForm from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.models import MetaData -from onadata.apps.messaging.constants import \ - XFORM, SUBMISSION_EDITED, SUBMISSION_CREATED +from onadata.apps.messaging.constants import ( + XFORM, + SUBMISSION_EDITED, + SUBMISSION_CREATED, +) from onadata.libs.utils import csv_import from onadata.libs.utils.common_tags import IMPORTED_VIA_CSV_BY from onadata.libs.utils.csv_import import get_submission_meta_dict @@ -26,143 +28,149 @@ def strip_xml_uuid(s): - return re.sub(b'\S*uuid\S*', b'', s.rstrip(b'\n')) # noqa + return re.sub(b"\S*uuid\S*", b"", s.rstrip(b"\n")) # noqa class CSVImportTestCase(TestBase): def setUp(self): super(CSVImportTestCase, self).setUp() - self.fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'libs', - 'tests', 'utils', 'fixtures') - self.good_csv = open(os.path.join(self.fixtures_dir, 'good.csv'), 'rb') - self.bad_csv = open(os.path.join(self.fixtures_dir, 'bad.csv'), 'rb') - self.xls_file_path = os.path.join(self.fixtures_dir, 'tutorial.xlsx') - self.good_xls = open( - os.path.join(self.fixtures_dir, 'good.xlsx'), 'rb') + self.fixtures_dir = os.path.join( + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures" + ) + self.good_csv = open(os.path.join(self.fixtures_dir, "good.csv"), "rb") + self.bad_csv = open(os.path.join(self.fixtures_dir, "bad.csv"), "rb") + self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial.xlsx") + self.good_xls = open(os.path.join(self.fixtures_dir, "good.xlsx"), "rb") def test_get_submission_meta_dict(self): self._publish_xls_file(self.xls_file_path) xform = XForm.objects.get() meta = get_submission_meta_dict(xform, None) self.assertEqual(len(meta), 2) - self.assertTrue('instanceID' in meta[0]) + self.assertTrue("instanceID" in meta[0]) self.assertEqual(meta[1], 0) - instance_id = 'uuid:9118a3fc-ab99-44cf-9a97-1bb1482d8e2b' + instance_id = "uuid:9118a3fc-ab99-44cf-9a97-1bb1482d8e2b" meta = get_submission_meta_dict(xform, instance_id) - self.assertTrue('instanceID' in meta[0]) - self.assertEqual(meta[0]['instanceID'], instance_id) + self.assertTrue("instanceID" in meta[0]) + self.assertEqual(meta[0]["instanceID"], instance_id) self.assertEqual(meta[1], 0) def test_submit_csv_param_sanity_check(self): - resp = csv_import.submit_csv('userX', XForm(), 123456) - self.assertIsNotNone(resp.get('error')) + resp = csv_import.submit_csv("userX", XForm(), 123456) + self.assertIsNotNone(resp.get("error")) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_xml_params(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = {} - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = BytesIO( - open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read()) + open(os.path.join(self.fixtures_dir, "single.xml"), "rb").read() + ) safe_create_args = list(safe_create_instance.call_args[0]) - self.assertEqual(safe_create_args[0], self.user.username, - 'Wrong username passed') + self.assertEqual( + safe_create_args[0], self.user.username, "Wrong username passed" + ) self.assertEqual( strip_xml_uuid(safe_create_args[1].getvalue()), strip_xml_uuid(xml_file_param.getvalue()), - 'Wrong xml param passed') - self.assertEqual(safe_create_args[2], [], - 'Wrong media array param passed') - self.assertEqual(safe_create_args[3], self.xform.uuid, - 'Wrong xform uuid passed') + "Wrong xml param passed", + ) + self.assertEqual(safe_create_args[2], [], "Wrong media array param passed") + self.assertEqual( + safe_create_args[3], self.xform.uuid, "Wrong xform uuid passed" + ) self.assertEqual(safe_create_args[4], None) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') - @mock.patch('onadata.libs.utils.csv_import.dict2xmlsubmission') - def test_submit_csv_xml_location_property_test(self, d2x, - safe_create_instance): + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") + @mock.patch("onadata.libs.utils.csv_import.dict2xmlsubmission") + def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = [ None, ] - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) - test_location_val = '83.3595 -32.8601 0 1' - test_location2_val = '21.22474 -10.5601 50000 200' + test_location_val = "83.3595 -32.8601 0 1" + test_location2_val = "21.22474 -10.5601 50000 200" - self.assertNotEqual(d2x.call_args, None, - 'dict2xmlsubmission not called') + self.assertNotEqual(d2x.call_args, None, "dict2xmlsubmission not called") call_dict = d2x.call_args[0][0] self.assertEqual( - call_dict.get('test_location'), test_location_val, - 'Location prop test fail') + call_dict.get("test_location"), test_location_val, "Location prop test fail" + ) self.assertEqual( - call_dict.get('test_location2'), test_location2_val, - 'Location2 prop test fail') + call_dict.get("test_location2"), + test_location2_val, + "Location2 prop test fail", + ) def test_submit_csv_and_rollback(self): - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), count + 9, - 'submit_csv test Failed!') + self.assertEqual(Instance.objects.count(), count + 9, "submit_csv test Failed!") # Check that correct # of submissions belong to our user self.assertEqual( Instance.objects.filter(user=self.user).count(), count + 8, - 'submit_csv username check failed!') + "submit_csv username check failed!", + ) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 9) # Test rollback on error and user feedback with patch( - 'onadata.libs.utils.csv_import.safe_create_instance' - ) as safe_create_mock: + "onadata.libs.utils.csv_import.safe_create_instance" + ) as safe_create_mock: safe_create_mock.side_effect = [AttributeError] initial_count = self.xform.num_of_submissions - resp = csv_import.submit_csv( - self.user.username, self.xform, self.good_csv) - self.assertEqual(resp, {'job_status': 'FAILURE'}) + resp = csv_import.submit_csv(self.user.username, self.xform, self.good_csv) + self.assertEqual(resp, {"job_status": "FAILURE"}) self.xform.refresh_from_db() - self.assertEqual( - initial_count, self.xform.num_of_submissions) + self.assertEqual(initial_count, self.xform.num_of_submissions) - @patch('onadata.libs.utils.logger_tools.send_message') + @patch("onadata.libs.utils.logger_tools.send_message") def test_submit_csv_edits(self, send_message_mock): - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) - edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv')) + edit_csv = open(os.path.join(self.fixtures_dir, "edit.csv")) edit_csv_str = edit_csv.read() edit_csv = BytesIO( edit_csv_str.format( - * [x.get('uuid') for x in Instance.objects.values('uuid')]) - .encode('utf-8')) + *[x.get("uuid") for x in Instance.objects.values("uuid")] + ).encode("utf-8") + ) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) - self.assertEqual(Instance.objects.count(), count, - 'submit_csv edits #2 test Failed!') + self.assertEqual( + Instance.objects.count(), count, "submit_csv edits #2 test Failed!" + ) # message sent upon submission edit self.assertTrue(send_message_mock.called) send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_EDITED) @@ -173,98 +181,102 @@ def test_import_non_utf8_csv(self): self.xform = XForm.objects.get() count = Instance.objects.count() - non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'), - 'rb') - result = csv_import.submit_csv(self.user.username, self.xform, - non_utf8_csv) + non_utf8_csv = open(os.path.join(self.fixtures_dir, "non_utf8.csv"), "rb") + result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv) + self.assertEqual( + result.get("error"), + "CSV file must be utf-8 encoded", + "Incorrect error message returned.", + ) self.assertEqual( - result.get('error'), 'CSV file must be utf-8 encoded', - 'Incorrect error message returned.') - self.assertEqual(Instance.objects.count(), count, - 'Non unicode csv import rollback failed!') + Instance.objects.count(), count, "Non unicode csv import rollback failed!" + ) def test_reject_spaces_in_headers(self): - xls_file_path = os.path.join(self.fixtures_dir, 'space_in_header.xlsx') + xls_file_path = os.path.join(self.fixtures_dir, "space_in_header.xlsx") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() - non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'), - 'rb') - result = csv_import.submit_csv(self.user.username, self.xform, - non_utf8csv) + non_utf8csv = open(os.path.join(self.fixtures_dir, "header_space.csv"), "rb") + result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv) self.assertEqual( - result.get('error'), - 'CSV file fieldnames should not contain spaces', - 'Incorrect error message returned.') + result.get("error"), + "CSV file fieldnames should not contain spaces", + "Incorrect error message returned.", + ) def test_nested_geo_paths_csv(self): - self.xls_file_path = os.path.join(self.fixtures_dir, - 'tutorial-nested-geo.xlsx') + self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial-nested-geo.xlsx") self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() - good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'), - 'rb') + good_csv = open(os.path.join(self.fixtures_dir, "another_good.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) def test_csv_with_multiple_select_in_one_column(self): - self.xls_file_path = os.path.join(self.fixtures_dir, - 'form_with_multiple_select.xlsx') + self.xls_file_path = os.path.join( + self.fixtures_dir, "form_with_multiple_select.xlsx" + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open( - os.path.join(self.fixtures_dir, - 'csv_import_with_multiple_select.csv'), - 'rb') + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, good_csv) - self.assertEqual(Instance.objects.count(), 1, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 1, "submit_csv edits #1 test Failed!" + ) def test_csv_imports_are_tracked(self): """ Test that submissions created via CSV Import are tracked """ - self.xls_file_path = os.path.join(self.fixtures_dir, - 'form_with_multiple_select.xlsx') + self.xls_file_path = os.path.join( + self.fixtures_dir, "form_with_multiple_select.xlsx" + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open( - os.path.join(self.fixtures_dir, - 'csv_import_with_multiple_select.csv'), - 'rb') + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 1) - self.assertEqual(Instance.objects.first().status, 'imported_via_csv') + self.assertEqual(Instance.objects.first().status, "imported_via_csv") def test_csv_imports_initiator_stored(self): """ Test that the user who imported data via CSV is tracked """ - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) - self.assertEqual(Instance.objects.count(), 9, - 'submit_csv edits #1 test Failed!') + self.assertEqual( + Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" + ) # Ensure bob user is tagged as the person who initiated the CSV Import metadata_qs = MetaData.objects.filter(data_type=IMPORTED_VIA_CSV_BY) self.assertEqual(metadata_qs.count(), 9) self.assertEqual(metadata_qs.first().data_value, self.user.username) def test_csv_with_repeats_import(self): - self.xls_file_path = os.path.join(self.this_directory, 'fixtures', - 'csv_export', - 'tutorial_w_repeats.xlsx') + self.xls_file_path = os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.xlsx" + ) repeats_csv = open( - os.path.join(self.this_directory, 'fixtures', 'csv_export', - 'tutorial_w_repeats.csv'), - 'rb') + os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.csv" + ), + "rb", + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() @@ -273,13 +285,18 @@ def test_csv_with_repeats_import(self): self.assertEqual(count, 1 + pre_count) def test_csv_with__more_than_4_repeats_import(self): - self.xls_file_path = os.path.join(self.this_directory, 'fixtures', - 'csv_export', - 'tutorial_w_repeats.xlsx') + self.xls_file_path = os.path.join( + self.this_directory, "fixtures", "csv_export", "tutorial_w_repeats.xlsx" + ) repeats_csv = open( - os.path.join(self.this_directory, 'fixtures', 'csv_export', - 'tutorial_w_repeats_import.csv'), - 'rb') + os.path.join( + self.this_directory, + "fixtures", + "csv_export", + "tutorial_w_repeats_import.csv", + ), + "rb", + ) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() @@ -290,14 +307,12 @@ def test_csv_with__more_than_4_repeats_import(self): instance = self.xform.instances.last() # repeats should be 6 - self.assertEqual(6, len(instance.json.get('children'))) + self.assertEqual(6, len(instance.json.get("children"))) - @mock.patch('onadata.libs.utils.csv_import.AsyncResult') + @mock.patch("onadata.libs.utils.csv_import.AsyncResult") def test_get_async_csv_submission_status(self, AsyncResult): result = csv_import.get_async_csv_submission_status(None) - self.assertEqual(result, - {'error': 'Empty job uuid', - 'job_status': 'FAILURE'}) + self.assertEqual(result, {"error": "Empty job uuid", "job_status": "FAILURE"}) class BacklogLimitExceededMockAsyncResult(object): def __init__(self): @@ -308,100 +323,104 @@ def state(self): raise BacklogLimitExceeded() AsyncResult.return_value = BacklogLimitExceededMockAsyncResult() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'PENDING'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "PENDING"}) class MockAsyncResult(object): def __init__(self): - self.result = self.state = 'SUCCESS' + self.result = self.state = "SUCCESS" def get(self): - return {'job_status': 'SUCCESS'} + return {"job_status": "SUCCESS"} AsyncResult.return_value = MockAsyncResult() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'SUCCESS'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "SUCCESS"}) class MockAsyncResult2(object): def __init__(self): - self.result = self.state = 'PROGRESS' + self.result = self.state = "PROGRESS" self.info = { "info": [], "job_status": "PROGRESS", "progress": 4000, - "total": 70605 + "total": 70605, } AsyncResult.return_value = MockAsyncResult2() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'info': [], 'job_status': 'PROGRESS', - 'progress': 4000, 'total': 70605}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual( + result, + {"info": [], "job_status": "PROGRESS", "progress": 4000, "total": 70605}, + ) class MockAsyncResultIOError(object): def __init__(self): self.result = IOError("File not found!") - self.state = 'FAILURE' + self.state = "FAILURE" AsyncResult.return_value = MockAsyncResultIOError() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, - {'error': 'File not found!', - 'job_status': 'FAILURE'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"error": "File not found!", "job_status": "FAILURE"}) # shouldn't fail if info is not of type dict class MockAsyncResult2(object): def __init__(self): - self.result = self.state = 'PROGRESS' + self.result = self.state = "PROGRESS" self.info = None AsyncResult.return_value = MockAsyncResult2() - result = csv_import.get_async_csv_submission_status('x-y-z') - self.assertEqual(result, {'job_status': 'PROGRESS'}) + result = csv_import.get_async_csv_submission_status("x-y-z") + self.assertEqual(result, {"job_status": "PROGRESS"}) def test_submission_xls_to_csv(self): """Test that submission_xls_to_csv converts to csv""" - c_csv_file = csv_import.submission_xls_to_csv( - self.good_xls) + c_csv_file = csv_import.submission_xls_to_csv(self.good_xls) c_csv_file.seek(0) - c_csv_reader = ucsv.DictReader(c_csv_file, encoding='utf-8-sig') - g_csv_reader = ucsv.DictReader(self.good_csv, encoding='utf-8-sig') + c_csv_reader = ucsv.DictReader(c_csv_file, encoding="utf-8-sig") + g_csv_reader = ucsv.DictReader(self.good_csv, encoding="utf-8-sig") - self.assertEqual( - g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10]) + self.assertEqual(g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10]) - @mock.patch('onadata.libs.utils.csv_import.safe_create_instance') + @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_instance_id_consistency(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = {} - single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "single.csv"), "rb") csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = BytesIO( - open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read()) + open(os.path.join(self.fixtures_dir, "single.xml"), "rb").read() + ) safe_create_args = list(safe_create_instance.call_args[0]) instance_xml = fromstring(safe_create_args[1].getvalue()) single_instance_xml = fromstring(xml_file_param.getvalue()) - instance_id = [ - m.find('instanceID').text for m in instance_xml.findall('meta')][0] - single_instance_id = [m.find('instanceID').text for m in - single_instance_xml.findall('meta')][0] + instance_id = [m.find("instanceID").text for m in instance_xml.findall("meta")][ + 0 + ] + single_instance_id = [ + m.find("instanceID").text for m in single_instance_xml.findall("meta") + ][0] self.assertEqual( - len(instance_id), len(single_instance_id), - "Same uuid length in generated xml") + len(instance_id), + len(single_instance_id), + "Same uuid length in generated xml", + ) - @patch('onadata.libs.utils.logger_tools.send_message') + @patch("onadata.libs.utils.logger_tools.send_message") def test_data_upload(self, send_message_mock): """Data upload for submissions with no uuids""" self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() - single_csv = open(os.path.join( - self.fixtures_dir, 'single_data_upload.csv'), 'rb') + single_csv = open( + os.path.join(self.fixtures_dir, "single_data_upload.csv"), "rb" + ) csv_import.submit_csv(self.user.username, self.xform, single_csv) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 1) @@ -428,44 +447,52 @@ def test_excel_date_conversion(self): """ self._create_user_and_login() - data = {'name': 'data'} + data = {"name": "data"} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) - survey['sms_keyword'] = survey['id_string'] + survey["sms_keyword"] = survey["id_string"] project = get_user_default_project(self.user) - xform = XForm(created_by=self.user, user=self.user, - xml=survey.to_xml(), json=survey.to_json(), - project=project) + xform = XForm( + created_by=self.user, + user=self.user, + xml=survey.to_xml(), + json=survey.to_json(), + project=project, + ) xform.save() - date_csv = open(os.path.join( - self.fixtures_dir, 'date.csv'), 'rb') + date_csv = open(os.path.join(self.fixtures_dir, "date.csv"), "rb") date_csv.seek(0) - csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') + csv_reader = ucsv.DictReader(date_csv, encoding="utf-8-sig") xl_dates = [] xl_datetime = [] # xl dates for row in csv_reader: - xl_dates.append(row.get('tdate')) - xl_datetime.append(row.get('now')) + xl_dates.append(row.get("tdate")) + xl_datetime.append(row.get("now")) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates - conv_dates = [instance.json.get('tdate') - for instance in Instance.objects.filter( - xform=xform).order_by('date_created')] - conv_datetime = [instance.json.get('now') - for instance in Instance.objects.filter( - xform=xform).order_by('date_created')] - - self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) + conv_dates = [ + instance.json.get("tdate") + for instance in Instance.objects.filter(xform=xform).order_by( + "date_created" + ) + ] + conv_datetime = [ + instance.json.get("now") + for instance in Instance.objects.filter(xform=xform).order_by( + "date_created" + ) + ] + + self.assertEqual(xl_dates, ["3/1/2019", "2/26/2019"]) self.assertEqual( - xl_datetime, - [u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00']) + xl_datetime, ["6/12/2020 13:20", "2019-03-11T16:00:51.147+02:00"] + ) self.assertEqual( - conv_datetime, - [u'2020-06-12T13:20:00', - u'2019-03-11T16:00:51.147000+02:00']) - self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26']) + conv_datetime, ["2020-06-12T13:20:00", "2019-03-11T16:00:51.147000+02:00"] + ) + self.assertEqual(conv_dates, ["2019-03-01", "2019-02-26"]) def test_enforces_data_type_and_rollback(self): """ @@ -473,17 +500,15 @@ def test_enforces_data_type_and_rollback(self): during the process are rolled back """ # Test integer constraint is enforced - xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", - "tests", "fixtures", "tutorial.xlsx") + xls_file_path = os.path.join( + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" + ) self._publish_xls_file(xls_file_path) self.xform = XForm.objects.last() - bad_data = open( - os.path.join(self.fixtures_dir, 'bad_data.csv'), - 'rb') + bad_data = open(os.path.join(self.fixtures_dir, "bad_data.csv"), "rb") count = Instance.objects.count() - result = csv_import.submit_csv(self.user.username, self.xform, - bad_data) + result = csv_import.submit_csv(self.user.username, self.xform, bad_data) expected_error = ( "Invalid CSV data imported in row(s): {1: ['Unknown date format(s)" @@ -492,21 +517,23 @@ def test_enforces_data_type_and_rollback(self): "(s): 22.32'], 4: ['Unknown date format(s): 2014-0900'], " "5: ['Unknown date format(s): 2014-0901'], 6: ['Unknown " "date format(s): 2014-0902'], 7: ['Unknown date format(s):" - " 2014-0903']}") - self.assertEqual( - result.get('error'), - expected_error) + " 2014-0903']}" + ) + self.assertEqual(result.get("error"), expected_error) # Assert all created instances were rolled back self.assertEqual(count, Instance.objects.count()) def test_csv_import_with_overwrite(self): self._publish_xls_file(self.xls_file_path) - surveys = ['uuid1'] + surveys = ["uuid1"] - paths = [os.path.join( - self.fixtures_dir, 'tutorial', 'instances', s, 'submission.xml') - for s in surveys] + paths = [ + os.path.join( + self.fixtures_dir, "tutorial", "instances", s, "submission.xml" + ) + for s in surveys + ] for path in paths: self._make_submission(path) @@ -516,11 +543,11 @@ def test_csv_import_with_overwrite(self): self.assertEqual(count, 1) - single_csv = open(os.path.join(self.fixtures_dir, 'same_uuid.csv'), - 'rb') + single_csv = open(os.path.join(self.fixtures_dir, "same_uuid.csv"), "rb") - csv_import.submit_csv(self.user.username, self.xform, single_csv, - overwrite=True) + csv_import.submit_csv( + self.user.username, self.xform, single_csv, overwrite=True + ) self.xform.refresh_from_db() count = self.xform.instances.filter(deleted_at=None).count() @@ -536,21 +563,18 @@ def test_get_columns_by_type(self): ) self._publish_xls_file(self.xls_file_path) xform = XForm.objects.get() - columns = get_columns_by_type(["date"], json.loads(xform.json)) + columns = get_columns_by_type(["date"], xform.json_dict()) self.assertEqual( columns, ["section_A/date_of_survey", "section_B/year_established"] ) good_csv = open( - os.path.join( - self.fixtures_dir, "csv_import_with_multiple_select.csv" - ), "rb" + os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"), "rb" ) csv_import.submit_csv(self.user.username, xform, good_csv) self.assertEqual(Instance.objects.count(), 1) submission = Instance.objects.first() self.assertEqual(submission.status, "imported_via_csv") - self.assertEqual(submission.json["section_A/date_of_survey"], - "2015-09-10") + self.assertEqual(submission.json["section_A/date_of_survey"], "2015-09-10") self.assertTrue( submission.json["section_B/year_established"].startswith("1890") ) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 9398a1506b..ca6b8f3cf3 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -576,8 +576,9 @@ def response_for_format(data, format=None): formatted_data = data.xls else: - formatted_data = json.loads(data.json) \ - if isinstance(data.json, str) else data.json + formatted_data = ( + json.loads(data.json) if isinstance(data.json, str) else data.json + ) return Response(formatted_data) diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 9207321335..9361cc7300 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -1,3 +1,4 @@ +import json import os import re import sys @@ -16,13 +17,20 @@ from dict2xml import dict2xml from django.conf import settings from django.contrib.auth.models import User -from django.core.exceptions import (MultipleObjectsReturned, PermissionDenied, - ValidationError) +from django.core.exceptions import ( + MultipleObjectsReturned, + PermissionDenied, + ValidationError, +) from django.core.files.storage import get_storage_class from django.db import IntegrityError, transaction, DataError from django.db.models import Q -from django.http import (HttpResponse, HttpResponseNotFound, - StreamingHttpResponse, UnreadablePostError) +from django.http import ( + HttpResponse, + HttpResponseNotFound, + StreamingHttpResponse, + UnreadablePostError, +) from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.encoding import DjangoUnicodeDecodeError @@ -33,19 +41,33 @@ from pyxform.xform2json import create_survey_element_from_xml from rest_framework.response import Response -from onadata.apps.logger.models import ( - Attachment, Instance, XForm, XFormVersion) +from onadata.apps.logger.models import Attachment, Instance, XForm, XFormVersion from onadata.apps.logger.models.instance import ( - FormInactiveError, InstanceHistory, FormIsMergedDatasetError, - get_id_string_from_xml_str) + FormInactiveError, + InstanceHistory, + FormIsMergedDatasetError, + get_id_string_from_xml_str, +) from onadata.apps.logger.models.xform import XLSFormError from onadata.apps.logger.xform_instance_parser import ( - DuplicateInstance, InstanceEmptyError, InstanceInvalidUserError, - InstanceMultipleNodeError, InstanceEncryptionError, NonUniqueFormIdError, - InstanceFormatError, clean_and_parse_xml, get_deprecated_uuid_from_xml, - get_submission_date_from_xml, get_uuid_from_xml, AttachmentNameError) -from onadata.apps.messaging.constants import XFORM, \ - SUBMISSION_EDITED, SUBMISSION_CREATED + DuplicateInstance, + InstanceEmptyError, + InstanceInvalidUserError, + InstanceMultipleNodeError, + InstanceEncryptionError, + NonUniqueFormIdError, + InstanceFormatError, + clean_and_parse_xml, + get_deprecated_uuid_from_xml, + get_submission_date_from_xml, + get_uuid_from_xml, + AttachmentNameError, +) +from onadata.apps.messaging.constants import ( + XFORM, + SUBMISSION_EDITED, + SUBMISSION_CREATED, +) from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance @@ -56,19 +78,21 @@ from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.user_auth import get_user_default_project -OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version' -HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION' -OPEN_ROSA_VERSION = '1.0' -DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8' +OPEN_ROSA_VERSION_HEADER = "X-OpenRosa-Version" +HTTP_OPEN_ROSA_VERSION_HEADER = "HTTP_X_OPENROSA_VERSION" +OPEN_ROSA_VERSION = "1.0" +DEFAULT_CONTENT_TYPE = "text/xml; charset=utf-8" DEFAULT_CONTENT_LENGTH = settings.DEFAULT_CONTENT_LENGTH REQUIRED_ENCRYPTED_FILE_ELEMENTS = [ "{http://www.opendatakit.org/xforms/encrypted}base64EncryptedKey", "{http://www.opendatakit.org/xforms/encrypted}encryptedXmlFile", "{http://opendatakit.org/submissions}base64EncryptedKey", - "{http://opendatakit.org/submissions}encryptedXmlFile"] + "{http://opendatakit.org/submissions}encryptedXmlFile", +] -uuid_regex = re.compile(r'\s*\s*([^<]+)\s*\s*', - re.DOTALL) +uuid_regex = re.compile( + r"\s*\s*([^<]+)\s*\s*", re.DOTALL +) def create_xform_version(xform: XForm, user: User) -> XFormVersion: @@ -80,28 +104,32 @@ def create_xform_version(xform: XForm, user: User) -> XFormVersion: return XFormVersion.objects.create( xform=xform, xls=xform.xls, - json=xform.json, + json=xform.json + if isinstance(xform.json, str) + else json.dumps(xform.json), version=xform.version, created_by=user, - xml=xform.xml + xml=xform.xml, ) except IntegrityError: pass -def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, - request=None): +def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, request=None): history = None instance = None message_verb = SUBMISSION_EDITED # check if its an edit submission old_uuid = get_deprecated_uuid_from_xml(xml) if old_uuid: - instance = Instance.objects.filter(uuid=old_uuid, - xform_id=xform.pk).first() - history = InstanceHistory.objects.filter( - xform_instance__xform_id=xform.pk, - uuid=new_uuid).only('xform_instance').first() + instance = Instance.objects.filter(uuid=old_uuid, xform_id=xform.pk).first() + history = ( + InstanceHistory.objects.filter( + xform_instance__xform_id=xform.pk, uuid=new_uuid + ) + .only("xform_instance") + .first() + ) if instance: # edits @@ -115,7 +143,8 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, uuid=old_uuid, user=submitted_by, geom=instance.geom, - submission_date=instance.last_edited or instance.date_created) + submission_date=instance.last_edited or instance.date_created, + ) instance.xml = xml instance.last_edited = last_edited instance.uuid = new_uuid @@ -123,22 +152,24 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, instance.save() # call webhooks - process_submission.send(sender=instance.__class__, - instance=instance) + process_submission.send(sender=instance.__class__, instance=instance) elif history: instance = history.xform_instance if old_uuid is None or (instance is None and history is None): # new submission message_verb = SUBMISSION_CREATED instance = Instance.objects.create( - xml=xml, user=submitted_by, status=status, xform=xform, - checksum=checksum) + xml=xml, user=submitted_by, status=status, xform=xform, checksum=checksum + ) # send notification on submission creation send_message( - instance_id=instance.id, target_id=instance.xform.id, - target_type=XFORM, user=instance.user or instance.xform.user, - message_verb=message_verb) + instance_id=instance.id, + target_id=instance.xform.id, + target_type=XFORM, + user=instance.user or instance.xform.user, + message_verb=message_verb, + ) return instance @@ -168,18 +199,18 @@ def dict2xform(jsform, form_id, root=None, username=None, gen_uuid=False): form = XForm.objects.filter( id_string__iexact=form_id, user__username__iexact=username, - deleted_at__isnull=True).first() - root = form.survey.name if form else 'data' + deleted_at__isnull=True, + ).first() + root = form.survey.name if form else "data" else: - root = 'data' + root = "data" if gen_uuid: - jsform['meta'] = { - 'instanceID': 'uuid:' + get_uuid(hex_only=False) - } + jsform["meta"] = {"instanceID": "uuid:" + get_uuid(hex_only=False)} return "<{0} id='{1}'>{2}".format( - root, form_id, dict2xml(jsform)) + root, form_id, dict2xml(jsform) + ) def get_first_record(queryset): @@ -195,14 +226,13 @@ def get_first_record(queryset): def get_uuid_from_submission(xml): # parse UUID from uploaded XML - split_xml = uuid_regex.split(xml.decode('utf-8')) + split_xml = uuid_regex.split(xml.decode("utf-8")) # check that xml has UUID return len(split_xml) > 1 and split_xml[1] or None -def get_xform_from_submission( - xml, username, uuid=None, request=None): +def get_xform_from_submission(xml, username, uuid=None, request=None): """Gets the submissions target XForm. Retrieves the target XForm by either utilizing the `uuid` param @@ -223,8 +253,7 @@ def get_xform_from_submission( if uuid: # try find the form by its uuid which is the ideal condition - if XForm.objects.filter( - uuid=uuid, deleted_at__isnull=True).count() > 0: + if XForm.objects.filter(uuid=uuid, deleted_at__isnull=True).count() > 0: xform = XForm.objects.get(uuid=uuid, deleted_at__isnull=True) # If request is present, verify that the request user # has the correct permissions @@ -246,17 +275,18 @@ def get_xform_from_submission( id_string = get_id_string_from_xml_str(xml) try: return get_object_or_404( - XForm, - id_string__iexact=id_string, - user__username__iexact=username, - deleted_at__isnull=True) + XForm, + id_string__iexact=id_string, + user__username__iexact=username, + deleted_at__isnull=True, + ) except MultipleObjectsReturned: raise NonUniqueFormIdError() def _has_edit_xform_permission(xform, user): if isinstance(xform, XForm) and isinstance(user, User): - return user.has_perm('logger.change_xform', xform) + return user.has_perm("logger.change_xform", xform) return False @@ -268,12 +298,16 @@ def check_edit_submission_permissions(request_user, xform): if requires_auth and not has_edit_perms: raise PermissionDenied( - _(u"%(request_user)s is not allowed to make edit submissions " - u"to %(form_user)s's %(form_title)s form." % { - 'request_user': request_user, - 'form_user': xform.user, - 'form_title': xform.title - })) + _( + "%(request_user)s is not allowed to make edit submissions " + "to %(form_user)s's %(form_title)s form." + % { + "request_user": request_user, + "form_user": xform.user, + "form_title": xform.title, + } + ) + ) def check_submission_permissions(request, xform): @@ -290,17 +324,27 @@ def check_submission_permissions(request, xform): :returns: None. :raises: PermissionDenied based on the above criteria. """ - if request and (xform.user.profile.require_auth or xform.require_auth or - request.path == '/submission')\ - and xform.user != request.user\ - and not request.user.has_perm('report_xform', xform): + if ( + request + and ( + xform.user.profile.require_auth + or xform.require_auth + or request.path == "/submission" + ) + and xform.user != request.user + and not request.user.has_perm("report_xform", xform) + ): raise PermissionDenied( - _(u"%(request_user)s is not allowed to make submissions " - u"to %(form_user)s's %(form_title)s form." % { - 'request_user': request.user, - 'form_user': xform.user, - 'form_title': xform.title - })) + _( + "%(request_user)s is not allowed to make submissions " + "to %(form_user)s's %(form_title)s form." + % { + "request_user": request.user, + "form_user": xform.user, + "form_title": xform.title, + } + ) + ) def check_submission_encryption(xform: XForm, xml: bytes) -> NoReturn: @@ -313,23 +357,26 @@ def check_submission_encryption(xform: XForm, xml: bytes) -> NoReturn: """ submission_encrypted = False submission_element = ET.fromstring(xml) - encrypted_attrib = submission_element.attrib.get('encrypted') + encrypted_attrib = submission_element.attrib.get("encrypted") required_encryption_elems = [ - elem.tag for elem in submission_element - if elem.tag in REQUIRED_ENCRYPTED_FILE_ELEMENTS] + elem.tag + for elem in submission_element + if elem.tag in REQUIRED_ENCRYPTED_FILE_ELEMENTS + ] encryption_elems_num = len(required_encryption_elems) # Check the validity of the submission if encrypted_attrib == "yes" or encryption_elems_num > 1: - if (not encryption_elems_num == 2 or - not encrypted_attrib == "yes") and xform.encrypted: - raise InstanceFormatError( - _("Encrypted submission incorrectly formatted.")) + if ( + not encryption_elems_num == 2 or not encrypted_attrib == "yes" + ) and xform.encrypted: + raise InstanceFormatError(_("Encrypted submission incorrectly formatted.")) submission_encrypted = True if xform.encrypted and not submission_encrypted: - raise InstanceEncryptionError(_( - "Unencrypted submissions are not allowed for encrypted forms.")) + raise InstanceEncryptionError( + _("Unencrypted submissions are not allowed for encrypted forms.") + ) def update_attachment_tracking(instance): @@ -339,8 +386,9 @@ def update_attachment_tracking(instance): instance.total_media = instance.num_of_media instance.media_count = instance.attachments_count instance.media_all_received = instance.media_count == instance.total_media - instance.save(update_fields=['total_media', 'media_count', - 'media_all_received', 'json']) + instance.save( + update_fields=["total_media", "media_count", "media_all_received", "json"] + ) def save_attachments(xform, instance, media_files, remove_deleted_media=False): @@ -351,9 +399,8 @@ def save_attachments(xform, instance, media_files, remove_deleted_media=False): for f in media_files: filename, extension = os.path.splitext(f.name) - extension = extension.replace('.', '') - content_type = u'text/xml' \ - if extension == Attachment.OSM else f.content_type + extension = extension.replace(".", "") + content_type = "text/xml" if extension == Attachment.OSM else f.content_type if extension == Attachment.OSM and not xform.instances_with_osm: xform.instances_with_osm = True xform.save() @@ -361,43 +408,51 @@ def save_attachments(xform, instance, media_files, remove_deleted_media=False): # Validate Attachment file name length if len(filename) > 100: raise AttachmentNameError(filename) - media_in_submission = ( - filename in instance.get_expected_media() or - [instance.xml.decode('utf-8').find(filename) != -1 if - isinstance(instance.xml, bytes) else - instance.xml.find(filename) != -1]) + media_in_submission = filename in instance.get_expected_media() or [ + instance.xml.decode("utf-8").find(filename) != -1 + if isinstance(instance.xml, bytes) + else instance.xml.find(filename) != -1 + ] if media_in_submission: Attachment.objects.get_or_create( instance=instance, media_file=f, mimetype=content_type, name=filename, - extension=extension) + extension=extension, + ) if remove_deleted_media: instance.soft_delete_attachments() update_attachment_tracking(instance) -def save_submission(xform, xml, media_files, new_uuid, submitted_by, status, - date_created_override, checksum, request=None): +def save_submission( + xform, + xml, + media_files, + new_uuid, + submitted_by, + status, + date_created_override, + checksum, + request=None, +): if not date_created_override: date_created_override = get_submission_date_from_xml(xml) - instance = _get_instance(xml, new_uuid, submitted_by, status, xform, - checksum, request) - save_attachments( - xform, - instance, - media_files, - remove_deleted_media=True) + instance = _get_instance( + xml, new_uuid, submitted_by, status, xform, checksum, request + ) + save_attachments(xform, instance, media_files, remove_deleted_media=True) # override date created if required if date_created_override: if not timezone.is_aware(date_created_override): # default to utc? date_created_override = timezone.make_aware( - date_created_override, timezone.utc) + date_created_override, timezone.utc + ) instance.date_created = date_created_override instance.save() @@ -416,13 +471,15 @@ def get_filtered_instances(*args, **kwargs): return Instance.objects.filter(*args, **kwargs) -def create_instance(username, - xml_file, - media_files, - status=u'submitted_via_web', - uuid=None, - date_created_override=None, - request=None): +def create_instance( + username, + xml_file, + media_files, + status="submitted_via_web", + uuid=None, + date_created_override=None, + request=None, +): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create @@ -433,8 +490,7 @@ def create_instance(username, * If there is a username and a uuid, submitting a new ODK form. """ instance = None - submitted_by = request.user \ - if request and request.user.is_authenticated else None + submitted_by = request.user if request and request.user.is_authenticated else None if username: username = username.lower() @@ -447,18 +503,16 @@ def create_instance(username, new_uuid = get_uuid_from_xml(xml) filtered_instances = get_filtered_instances( - Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk) - existing_instance = get_first_record(filtered_instances.only('id')) - if existing_instance and \ - (new_uuid or existing_instance.xform.has_start_time): + Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk + ) + existing_instance = get_first_record(filtered_instances.only("id")) + if existing_instance and (new_uuid or existing_instance.xform.has_start_time): # ensure we have saved the extra attachments with transaction.atomic(): save_attachments( - xform, - existing_instance, - media_files, - remove_deleted_media=True) - existing_instance.save(update_fields=['json', 'date_modified']) + xform, existing_instance, media_files, remove_deleted_media=True + ) + existing_instance.save(update_fields=["json", "date_modified"]) # Ignore submission as a duplicate IFF # * a submission's XForm collects start time @@ -467,20 +521,23 @@ def create_instance(username, return DuplicateInstance() # get new and deprecated UUIDs - history = InstanceHistory.objects.filter( - xform_instance__xform_id=xform.pk, - xform_instance__deleted_at__isnull=True, - uuid=new_uuid).only('xform_instance').first() + history = ( + InstanceHistory.objects.filter( + xform_instance__xform_id=xform.pk, + xform_instance__deleted_at__isnull=True, + uuid=new_uuid, + ) + .only("xform_instance") + .first() + ) if history: duplicate_instance = history.xform_instance # ensure we have saved the extra attachments with transaction.atomic(): save_attachments( - xform, - duplicate_instance, - media_files, - remove_deleted_media=True) + xform, duplicate_instance, media_files, remove_deleted_media=True + ) duplicate_instance.save() return DuplicateInstance() @@ -488,23 +545,31 @@ def create_instance(username, try: with transaction.atomic(): if isinstance(xml, bytes): - xml = xml.decode('utf-8') - instance = save_submission(xform, xml, media_files, new_uuid, - submitted_by, status, - date_created_override, checksum, - request) + xml = xml.decode("utf-8") + instance = save_submission( + xform, + xml, + media_files, + new_uuid, + submitted_by, + status, + date_created_override, + checksum, + request, + ) except IntegrityError: - instance = get_first_record(Instance.objects.filter( - Q(checksum=checksum) | Q(uuid=new_uuid), - xform_id=xform.pk)) + instance = get_first_record( + Instance.objects.filter( + Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk + ) + ) if instance: attachment_names = [ - a.media_file.name.split('/')[-1] + a.media_file.name.split("/")[-1] for a in Attachment.objects.filter(instance=instance) ] - media_files = [f for f in media_files - if f.name not in attachment_names] + media_files = [f for f in media_files if f.name not in attachment_names] save_attachments(xform, instance, media_files) instance.save() @@ -514,8 +579,13 @@ def create_instance(username, @use_master def safe_create_instance( - username, xml_file, media_files, uuid, request, - instance_status: str = 'submitted_via_web'): + username, + xml_file, + media_files, + uuid, + request, + instance_status: str = "submitted_via_web", +): """Create an instance and catch exceptions. :returns: A list [error, instance] where error is None if there was no @@ -525,13 +595,19 @@ def safe_create_instance( try: instance = create_instance( - username, xml_file, media_files, uuid=uuid, request=request, - status=instance_status) + username, + xml_file, + media_files, + uuid=uuid, + request=request, + status=instance_status, + ) except InstanceInvalidUserError: - error = OpenRosaResponseBadRequest(_(u"Username or ID required.")) + error = OpenRosaResponseBadRequest(_("Username or ID required.")) except InstanceEmptyError: error = OpenRosaResponseBadRequest( - _(u"Received empty submission. No instance was created")) + _("Received empty submission. No instance was created") + ) except InstanceEncryptionError as e: error = OpenRosaResponseBadRequest(text(e)) except InstanceFormatError as e: @@ -539,56 +615,58 @@ def safe_create_instance( except (FormInactiveError, FormIsMergedDatasetError) as e: error = OpenRosaResponseNotAllowed(text(e)) except XForm.DoesNotExist: - error = OpenRosaResponseNotFound( - _(u"Form does not exist on this account")) + error = OpenRosaResponseNotFound(_("Form does not exist on this account")) except ExpatError: - error = OpenRosaResponseBadRequest(_(u"Improperly formatted XML.")) + error = OpenRosaResponseBadRequest(_("Improperly formatted XML.")) except AttachmentNameError: response = OpenRosaResponseBadRequest( - _("Attachment file name exceeds 100 chars")) + _("Attachment file name exceeds 100 chars") + ) response.status_code = 400 error = response except DuplicateInstance: - response = OpenRosaResponse(_(u"Duplicate submission")) + response = OpenRosaResponse(_("Duplicate submission")) response.status_code = 202 if request: - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) error = response except PermissionDenied as e: error = OpenRosaResponseForbidden(e) except UnreadablePostError as e: error = OpenRosaResponseBadRequest( - _(u"Unable to read submitted file: %(error)s" - % {'error': text(e)})) + _("Unable to read submitted file: %(error)s" % {"error": text(e)}) + ) except InstanceMultipleNodeError as e: error = OpenRosaResponseBadRequest(e) except DjangoUnicodeDecodeError: error = OpenRosaResponseBadRequest( - _(u"File likely corrupted during " - u"transmission, please try later.")) + _("File likely corrupted during " "transmission, please try later.") + ) except NonUniqueFormIdError: error = OpenRosaResponseBadRequest( - _(u"Unable to submit because there are multiple forms with" - u" this formID.")) + _("Unable to submit because there are multiple forms with" " this formID.") + ) except DataError as e: error = OpenRosaResponseBadRequest((str(e))) if isinstance(instance, DuplicateInstance): - response = OpenRosaResponse(_(u"Duplicate submission")) + response = OpenRosaResponse(_("Duplicate submission")) response.status_code = 202 if request: - response['Location'] = request.build_absolute_uri(request.path) + response["Location"] = request.build_absolute_uri(request.path) error = response instance = None return [error, instance] -def response_with_mimetype_and_name(mimetype, - name, - extension=None, - show_date=True, - file_path=None, - use_local_filesystem=False, - full_mime=False): +def response_with_mimetype_and_name( + mimetype, + name, + extension=None, + show_date=True, + file_path=None, + use_local_filesystem=False, + full_mime=False, +): if extension is None: extension = mimetype if not full_mime: @@ -598,30 +676,28 @@ def response_with_mimetype_and_name(mimetype, if not use_local_filesystem: default_storage = get_storage_class()() wrapper = FileWrapper(default_storage.open(file_path)) - response = StreamingHttpResponse( - wrapper, content_type=mimetype) - response['Content-Length'] = default_storage.size(file_path) + response = StreamingHttpResponse(wrapper, content_type=mimetype) + response["Content-Length"] = default_storage.size(file_path) else: wrapper = FileWrapper(open(file_path)) - response = StreamingHttpResponse( - wrapper, content_type=mimetype) - response['Content-Length'] = os.path.getsize(file_path) + response = StreamingHttpResponse(wrapper, content_type=mimetype) + response["Content-Length"] = os.path.getsize(file_path) except IOError: - response = HttpResponseNotFound( - _(u"The requested file could not be found.")) + response = HttpResponseNotFound(_("The requested file could not be found.")) else: response = HttpResponse(content_type=mimetype) - response['Content-Disposition'] = generate_content_disposition_header( - name, extension, show_date) + response["Content-Disposition"] = generate_content_disposition_header( + name, extension, show_date + ) return response def generate_content_disposition_header(name, extension, show_date=True): if name is None: - return 'attachment;' + return "attachment;" if show_date: name = "%s-%s" % (name, datetime.now().strftime("%Y-%m-%d-%H-%M-%S")) - return 'attachment; filename=%s.%s' % (name, extension) + return "attachment; filename=%s.%s" % (name, extension) def store_temp_file(data): @@ -644,37 +720,36 @@ def publish_form(callback): try: return callback() except (PyXFormError, XLSFormError) as e: - return {'type': 'alert-error', 'text': text(e)} + return {"type": "alert-error", "text": text(e)} except IntegrityError: return { - 'type': 'alert-error', - 'text': _(u'Form with this id or SMS-keyword already exists.'), + "type": "alert-error", + "text": _("Form with this id or SMS-keyword already exists."), } except ProcessTimedOut: # catch timeout errors return { - 'type': 'alert-error', - 'text': _(u'Form validation timeout, please try again.'), + "type": "alert-error", + "text": _("Form validation timeout, please try again."), } except (MemoryError, OSError, BadStatusLine): return { - 'type': 'alert-error', - 'text': _((u'An error occurred while publishing the form. ' - 'Please try again.')), + "type": "alert-error", + "text": _( + ("An error occurred while publishing the form. " "Please try again.") + ), } except (AttributeError, Exception, ValidationError) as e: - report_exception("Form publishing exception: {}".format(e), text(e), - sys.exc_info()) - return {'type': 'alert-error', 'text': text(e)} + report_exception( + "Form publishing exception: {}".format(e), text(e), sys.exc_info() + ) + return {"type": "alert-error", "text": text(e)} @track_object_event( - user_field='user', - properties={ - 'created_by': 'user', - 'xform_id': 'pk', - 'xform_name': 'title'}, - additional_context={'from': 'Publish XLS Form'} + user_field="user", + properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, + additional_context={"from": "Publish XLS Form"}, ) @transaction.atomic() def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): @@ -683,16 +758,13 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): """ # get or create DataDictionary based on user and id string if id_string: - dd = DataDictionary.objects.get( - user=user, id_string=id_string, project=project) + dd = DataDictionary.objects.get(user=user, id_string=id_string, project=project) dd.xls = xls_file dd.save() else: dd = DataDictionary.objects.create( - created_by=created_by or user, - user=user, - xls=xls_file, - project=project) + created_by=created_by or user, user=user, xls=xls_file, project=project + ) # Create an XFormVersion object for the published XLSForm create_xform_version(dd, user) @@ -700,22 +772,18 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): @track_object_event( - user_field='user', - properties={ - 'created_by': 'user', - 'xform_id': 'pk', - 'xform_name': 'title'}, - additional_context={'from': 'Publish XML Form'} + user_field="user", + properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, + additional_context={"from": "Publish XML Form"}, ) def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): xml = xml_file.read() if isinstance(xml, bytes): - xml = xml.decode('utf-8') + xml = xml.decode("utf-8") survey = create_survey_element_from_xml(xml) form_json = survey.to_json() if id_string: - dd = DataDictionary.objects.get( - user=user, id_string=id_string, project=project) + dd = DataDictionary.objects.get(user=user, id_string=id_string, project=project) dd.xml = xml dd.json = form_json dd._mark_start_time_boolean() @@ -726,11 +794,8 @@ def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): else: created_by = created_by or user dd = DataDictionary( - created_by=created_by, - user=user, - xml=xml, - json=form_json, - project=project) + created_by=created_by, user=user, xml=xml, json=form_json, project=project + ) dd._mark_start_time_boolean() set_uuid(dd) dd._set_uuid_in_xml(file_name=xml_file.name) @@ -763,10 +828,10 @@ def __init__(self, *args, **kwargs): self[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') - self['Date'] = dt - self['X-OpenRosa-Accept-Content-Length'] = DEFAULT_CONTENT_LENGTH - self['Content-Type'] = DEFAULT_CONTENT_TYPE + dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") + self["Date"] = dt + self["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH + self["Content-Type"] = DEFAULT_CONTENT_TYPE class OpenRosaResponse(BaseOpenRosaResponse): @@ -776,10 +841,13 @@ def __init__(self, *args, **kwargs): super(OpenRosaResponse, self).__init__(*args, **kwargs) self.message = self.content # wrap content around xml - self.content = ''' + self.content = ( + """ %s -''' % self.content +""" + % self.content + ) class OpenRosaResponseNotFound(OpenRosaResponse): @@ -804,11 +872,11 @@ class OpenRosaNotAuthenticated(Response): def __init__(self, *args, **kwargs): super(OpenRosaNotAuthenticated, self).__init__(*args, **kwargs) - self['Content-Type'] = 'text/html; charset=utf-8' - self['X-OpenRosa-Accept-Content-Length'] = DEFAULT_CONTENT_LENGTH + self["Content-Type"] = "text/html; charset=utf-8" + self["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime('%a, %d %b %Y %H:%M:%S %Z') - self['Date'] = dt + dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") + self["Date"] = dt def inject_instanceid(xml_str, uuid): @@ -821,7 +889,8 @@ def inject_instanceid(xml_str, uuid): # check if we have a meta tag survey_node = children.item(0) meta_tags = [ - n for n in survey_node.childNodes + n + for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta" ] if len(meta_tags) == 0: @@ -832,7 +901,8 @@ def inject_instanceid(xml_str, uuid): # check if we have an instanceID tag uuid_tags = [ - n for n in meta_tag.childNodes + n + for n in meta_tag.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID" ] if len(uuid_tags) == 0: @@ -841,7 +911,7 @@ def inject_instanceid(xml_str, uuid): else: uuid_tag = uuid_tags[0] # insert meta and instanceID - text_node = xml.createTextNode(u"uuid:%s" % uuid) + text_node = xml.createTextNode("uuid:%s" % uuid) uuid_tag.appendChild(text_node) return xml.toxml() return xml_str From 22a40a1457707711960c8e43d9b3c4388bd709a4 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 19:48:17 +0300 Subject: [PATCH 200/234] sms_support: cleanup --- onadata/apps/sms_support/autodoc.py | 70 +++++---- onadata/apps/sms_support/parser.py | 112 ++++++++------ onadata/apps/sms_support/providers/textit.py | 142 ++++++++++-------- .../apps/sms_support/tests/test_base_sms.py | 58 ++++--- .../apps/sms_support/tests/test_notallowed.py | 14 +- onadata/apps/sms_support/tests/test_parser.py | 51 +++---- onadata/apps/sms_support/tools.py | 8 +- onadata/apps/sms_support/views.py | 76 ++++++---- 8 files changed, 291 insertions(+), 240 deletions(-) diff --git a/onadata/apps/sms_support/autodoc.py b/onadata/apps/sms_support/autodoc.py index 4084140cb8..36e959e641 100644 --- a/onadata/apps/sms_support/autodoc.py +++ b/onadata/apps/sms_support/autodoc.py @@ -24,7 +24,8 @@ ) -def get_sample_data_for(question, json_survey, as_names=False): +# pylint: disable=too-many-return-statements,too-many-branches +def get_sample_data_for(question, json_survey, as_names=False): # noqa C901 """return an example data for a particular question. If as_names is True, returns name (not sms_field) of the question""" @@ -49,32 +50,31 @@ def safe_wrap(value): if xlsf_type == "text": return safe_wrap("lorem ipsum") - elif xlsf_type == "integer": + if xlsf_type == "integer": return safe_wrap(4) - elif xlsf_type == "decimal": + if xlsf_type == "decimal": return safe_wrap(1.2) - elif xlsf_type == "select one": + if xlsf_type == "select one": return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:1])) - elif xlsf_type == "select all that apply": + if xlsf_type == "select all that apply": return safe_wrap(" ".join([c.get("sms_option") for c in xlsf_choices][:2])) - elif xlsf_type == "geopoint": + if xlsf_type == "geopoint": return safe_wrap("12.65 -8") - elif xlsf_type in MEDIA_TYPES: + if xlsf_type in MEDIA_TYPES: exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} - return safe_wrap("x.%s;dGhpc" % exts.get(xlsf_type, "ext")) - elif xlsf_type == "barcode": + return safe_wrap(f"x.{exts.get(xlsf_type, 'ext')};dGhpc") + if xlsf_type == "barcode": return safe_wrap("abc") - elif xlsf_type == "date": + if xlsf_type == "date": return safe_wrap(now.strftime(xlsf_date_fmt)) - elif xlsf_type == "datetime": + if xlsf_type == "datetime": return safe_wrap(now.strftime(xlsf_datetime_fmt)) - elif xlsf_type == "note": + if xlsf_type == "note": return None - else: - return safe_wrap("?") + return safe_wrap("?") -def get_helper_text(question, json_survey): +def get_helper_text(question, json_survey): # noqa C901 """The full sentence (html) of the helper for a question Includes the type, a description @@ -100,15 +100,15 @@ def safe_wrap(value, xlsf_type=xlsf_type): return text(value) if xlsf_type == "text": - return safe_wrap("Any string (excluding “%s”)" % separator) - elif xlsf_type == "integer": + return safe_wrap(f"Any string (excluding “{separator}”)") + if xlsf_type == "integer": return safe_wrap("Any integer digit.") - elif xlsf_type == "decimal": + if xlsf_type == "decimal": return safe_wrap("A decimal or integer value.") - elif xlsf_type == "select one": + if xlsf_type == "select one": helper = "Select one of the following:" helper += "
      " - # pylint: disable=E1101 + # pylint: disable=no-member helper += "".join( [ '
    • ' @@ -120,10 +120,10 @@ def safe_wrap(value, xlsf_type=xlsf_type): ) helper += "
    " return safe_wrap(helper) - elif xlsf_type == "select all that apply": + if xlsf_type == "select all that apply": helper = "Select none, one or more in:" helper += "
      " - # pylint: disable=E1101 + # pylint: disable=no-member helper += "".join( [ '
    • ' @@ -135,7 +135,7 @@ def safe_wrap(value, xlsf_type=xlsf_type): ) helper += "
    " return safe_wrap(helper) - elif xlsf_type == "geopoint": + if xlsf_type == "geopoint": helper = ( 'GPS coordinates as ' "latitude longitude." @@ -143,29 +143,29 @@ def safe_wrap(value, xlsf_type=xlsf_type): "altitude precision     after. All of them are decimal." ) return safe_wrap(helper) - elif xlsf_type in MEDIA_TYPES: + if xlsf_type in MEDIA_TYPES: exts = {"audio": "mp3", "video": "avi", "photo": "jpg"} helper = ( "File name and base64 data of the file as in " - 'x.%s;dGhpc.' + '' + f'x.{exts.get(xlsf_type, "ext")};dGhpc.' "
    It is not intented to be filled by " - "humans." % exts.get(xlsf_type, "ext") + "humans." ) return safe_wrap(helper) - elif xlsf_type == "barcode": + if xlsf_type == "barcode": return safe_wrap("A string representing the value behind the barcode.") - elif xlsf_type == "date": + if xlsf_type == "date": return safe_wrap( "A date in the format: " - '%s' % xlsf_date_fmt + f'{xlsf_date_fmt}' ) - elif xlsf_type == "datetime": + if xlsf_type == "datetime": return safe_wrap( "A datetime in the format: " - '%s' % xlsf_datetime_fmt + f'{xlsf_datetime_fmt}' ) - else: - return safe_wrap("?") + return safe_wrap("?") def get_autodoc_for(xform): @@ -190,9 +190,7 @@ def get_autodoc_for(xform): '%(keyword)s' "%(qid)d " % {"keyword": xform.sms_id_string, "qid": len(helpers)} ) - line_values = '%(keyword)s ' % { - "keyword": xform.sms_id_string - } + line_values = f'{xform.sms_id_string} ' helpers.append( ( "keyword", diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index b28d40a870..44ee14f598 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -1,46 +1,55 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 nu +""" +SMS parser module - utility functionality to process SMS messages. +""" import base64 +import binascii import logging import re -from builtins import str as text -from datetime import datetime, date +from datetime import date, datetime from io import BytesIO from django.utils.translation import gettext as _ from onadata.apps.logger.models import XForm from onadata.apps.sms_support.tools import ( + DEFAULT_DATE_FORMAT, + DEFAULT_DATETIME_FORMAT, + DEFAULT_SEPARATOR, + MEDIA_TYPES, + META_FIELDS, + NA_VALUE, SMS_API_ERROR, SMS_PARSING_ERROR, + SMS_SUBMISSION_ACCEPTED, SMS_SUBMISSION_REFUSED, - sms_media_to_file, generate_instance, - DEFAULT_SEPARATOR, - NA_VALUE, - META_FIELDS, - MEDIA_TYPES, - DEFAULT_DATE_FORMAT, - DEFAULT_DATETIME_FORMAT, - SMS_SUBMISSION_ACCEPTED, is_last, + sms_media_to_file, ) from onadata.libs.utils.logger_tools import dict2xform class SMSSyntaxError(ValueError): - pass + """A custom SMS syntax error exception class.""" class SMSCastingError(ValueError): + """A custom SMS type casting error exception class.""" + def __init__(self, message, question=None): if question: message = _("%(question)s: %(message)s") % { "question": question, "message": message, } - super(SMSCastingError, self).__init__(message) + super().__init__(message) -def parse_sms_text(xform, identity, text): +# pylint: disable=too-many-locals,too-many-branches +def parse_sms_text(xform, identity, sms_text): # noqa C901 + """Parses an SMS text to return XForm specific answers, media, notes.""" json_survey = xform.json_dict() @@ -58,13 +67,15 @@ def parse_sms_text(xform, identity, text): # extract SMS data into indexed groups of values groups = {} - for group in text.split(separator)[1:]: + for group in sms_text.split(separator)[1:]: group_id, group_text = [s.strip() for s in group.split(None, 1)] groups.update({group_id: [s.strip() for s in group_text.split(None)]}) - def cast_sms_value(value, question, medias=[]): + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches + def cast_sms_value(value, question, medias=None): """Check data type of value and return cleaned version""" + medias = [] if medias is None else medias xlsf_type = question.get("type") xlsf_name = question.get("name") xlsf_choices = question.get("children") @@ -76,14 +87,14 @@ def cast_sms_value(value, question, medias=[]): # unsafe. # xlsf_constraint=question.get('constraint') - if xlsf_required and not len(value): + if xlsf_required and not value: raise SMSCastingError(_("Required field missing"), xlsf_name) def safe_wrap(func): try: return func() except Exception as e: - raise SMSCastingError(_("%(error)s") % {"error": e}, xlsf_name) + raise SMSCastingError(_("%(error)s") % {"error": e}, xlsf_name) from e def media_value(value, medias): """handle media values @@ -94,26 +105,26 @@ def media_value(value, medias): filename, b64content = value.split(";", 1) medias.append((filename, base64.b64decode(b64content))) return filename - except Exception as e: + except (AttributeError, TypeError, binascii.Error) as e: raise SMSCastingError( _("Media file format " "incorrect. %(except)r") % {"except": e}, xlsf_name, - ) + ) from e if xlsf_type == "text": return safe_wrap(lambda: str(value)) - elif xlsf_type == "integer": + if xlsf_type == "integer": return safe_wrap(lambda: int(value)) - elif xlsf_type == "decimal": + if xlsf_type == "decimal": return safe_wrap(lambda: float(value)) - elif xlsf_type == "select one": + if xlsf_type == "select one": for choice in xlsf_choices: if choice.get("sms_option") == value: return choice.get("name") raise SMSCastingError( _("No matching choice " "for '%(input)s'") % {"input": value}, xlsf_name ) - elif xlsf_type == "select all that apply": + if xlsf_type == "select all that apply": values = [s.strip() for s in value.split()] ret_values = [] for indiv_value in values: @@ -121,7 +132,7 @@ def media_value(value, medias): if choice.get("sms_option") == indiv_value: ret_values.append(choice.get("name")) return " ".join(ret_values) - elif xlsf_type == "geopoint": + if xlsf_type == "geopoint": err_msg = _("Incorrect geopoint coordinates.") geodata = [s.strip() for s in value.split()] if len(geodata) < 2 and len(geodata) > 4: @@ -130,30 +141,30 @@ def media_value(value, medias): # check that latitude and longitude are floats lat, lon = [float(v) for v in geodata[:2]] # and within sphere boundaries - if lat < -90 or lat > 90 or lon < -180 and lon > 180: + if -90 > lat > 90 or -180 > lon > 180: raise SMSCastingError(err_msg, xlsf_name) if len(geodata) == 4: # check that altitude and accuracy are integers - [int(v) for v in geodata[2:4]] + for v in geodata[2:4]: + int(v) elif len(geodata) == 3: # check that altitude is integer int(geodata[2]) - except Exception as e: - raise SMSCastingError(e, xlsf_name) + except ValueError as e: + raise SMSCastingError(e, xlsf_name) from e return " ".join(geodata) - - elif xlsf_type in MEDIA_TYPES: + if xlsf_type in MEDIA_TYPES: # media content (image, video, audio) must be formatted as: # file_name;base64 encodeed content. # Example: hello.jpg;dGhpcyBpcyBteSBwaWN0dXJlIQ== return media_value(value, medias) - elif xlsf_type == "barcode": - return safe_wrap(lambda: text(value)) - elif xlsf_type == "date": + if xlsf_type == "barcode": + return safe_wrap(lambda: str(value)) + if xlsf_type == "date": return safe_wrap(lambda: datetime.strptime(value, xlsf_date_fmt).date()) - elif xlsf_type == "datetime": + if xlsf_type == "datetime": return safe_wrap(lambda: datetime.strptime(value, xlsf_datetime_fmt)) - elif xlsf_type == "note": + if xlsf_type == "note": return safe_wrap(lambda: "") raise SMSCastingError( _("Unsuported column '%(type)s'") % {"type": xlsf_type}, xlsf_name @@ -163,11 +174,11 @@ def get_meta_value(xlsf_type, identity): """XLSForm Meta field value""" if xlsf_type in ("deviceid", "subscriberid", "imei"): return NA_VALUE - elif xlsf_type in ("start", "end"): + if xlsf_type in ("start", "end"): return datetime.now().isoformat() - elif xlsf_type == "today": + if xlsf_type == "today": return date.today().isoformat() - elif xlsf_type == "phonenumber": + if xlsf_type == "phonenumber": return identity return NA_VALUE @@ -250,7 +261,8 @@ def get_meta_value(xlsf_type, identity): return survey_answers, medias, notes -def process_incoming_smses(username, incomings, id_string=None): +# pylint: disable=too-many-statements +def process_incoming_smses(username, incomings, id_string=None): # noqa C901 """Process Incoming (identity, text[, id_string]) SMS""" xforms = [] @@ -264,6 +276,7 @@ def process_incoming_smses(username, incomings, id_string=None): ) } + # pylint: disable=too-many-branches def process_incoming(incoming, id_string): # assign variables if len(incoming) >= 2: @@ -281,7 +294,7 @@ def process_incoming(incoming, id_string): ) return - if not len(identity.strip()) or not len(text.strip()): + if not identity.strip() or not text.strip(): responses.append( { "code": SMS_API_ERROR, @@ -349,8 +362,7 @@ def process_incoming(incoming, id_string): responses.append( { "code": SMS_SUBMISSION_REFUSED, - "text": _("Required field `%(field)s` is " "missing.") - % {"field": field}, + "text": _(f"Required field `{field}` is " "missing."), } ) return @@ -360,13 +372,13 @@ def process_incoming(incoming, id_string): # compute notes data = {} - for g in json_submission.values(): - data.update(g) + for group in json_submission.values(): + data.update(group) for idx, note in enumerate(notes): try: notes[idx] = note.replace("${", "{").format(**data) - except Exception as e: - logging.exception(_("Updating note threw exception: %s" % text(e))) + except AttributeError as e: + logging.exception("Updating note threw exception: %s", str(e)) # process_incoming expectes submission to be a file-like object xforms.append(BytesIO(xml_submission.encode("utf-8"))) @@ -377,8 +389,8 @@ def process_incoming(incoming, id_string): for incoming in incomings: try: process_incoming(incoming, id_string) - except Exception as e: - responses.append({"code": SMS_PARSING_ERROR, "text": text(e)}) + except (SMSCastingError, SMSSyntaxError, ValueError) as e: + responses.append({"code": SMS_PARSING_ERROR, "text": str(e)}) for idx, xform in enumerate(xforms): # generate_instance expects media as a request.FILES.values() list @@ -397,8 +409,8 @@ def process_incoming(incoming, id_string): # extend success_response with data from the answers data = {} - for g in json_submissions[idx].values(): - data.update(g) + for group in json_submissions[idx].values(): + data.update(group) success_response = success_response.replace("${", "{").format(**data) response.update({"text": success_response}) # add sendouts (notes) diff --git a/onadata/apps/sms_support/providers/textit.py b/onadata/apps/sms_support/providers/textit.py index 21b69efa55..3cb42443b4 100644 --- a/onadata/apps/sms_support/providers/textit.py +++ b/onadata/apps/sms_support/providers/textit.py @@ -8,127 +8,143 @@ See: https://textit.in/api/v1/webhook/ """ import datetime -import dateutil -import json -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.urls import reverse from django.utils.translation import gettext as _ -from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +import dateutil -from onadata.apps.sms_support.tools import SMS_API_ERROR,\ - SMS_SUBMISSION_ACCEPTED from onadata.apps.sms_support.parser import process_incoming_smses +from onadata.apps.sms_support.tools import SMS_API_ERROR, SMS_SUBMISSION_ACCEPTED -TEXTIT_URL = 'https://api.textit.in/api/v1/sms.json' +TEXTIT_URL = "https://api.textit.in/api/v1/sms.json" def autodoc(url_root, username, id_string): - urla = url_root + reverse('sms_submission_api', - kwargs={'username': username, - 'service': 'textit'}) - urlb = url_root + reverse('sms_submission_form_api', - kwargs={'username': username, - 'id_string': id_string, - 'service': 'textit'}) - doc = (u'

    ' + - _(u"%(service)s Instructions:") - % {'service': u'' - u'TextIt\'s Webhook API'} + - u'

    1. ' + - _(u"Sign in to TextIt.in and go to Account Page.") + - u'
    2. ' + - _(u"Tick “Incoming SMS Messages” and set Webhook URL to either:") + - u'
      %(urla)s' + - u'
      %(urlb)s
      ' + - u'

    ' + - _(u"That's it. Now Send an SMS Formhub submission to your TextIt" - u" phone number. It will create a submission on Formhub.") + - u'

    ') % {'urla': urla, 'urlb': urlb} + """Returns the documentation string for the provider.""" + urla = url_root + reverse( + "sms_submission_api", kwargs={"username": username, "service": "textit"} + ) + urlb = url_root + reverse( + "sms_submission_form_api", + kwargs={"username": username, "id_string": id_string, "service": "textit"}, + ) + doc = ( + "

    " + + _("%(service)s Instructions:") + % {"service": '' "TextIt's Webhook API"} + + "

    1. " + + _("Sign in to TextIt.in and go to Account Page.") + + "
    2. " + + _("Tick “Incoming SMS Messages” and set Webhook URL to either:") + + '
      %(urla)s' + + "
      %(urlb)s
      " + + "

    " + + _( + "That's it. Now Send an SMS Formhub submission to your TextIt" + " phone number. It will create a submission on Formhub." + ) + + "

    " + ) % {"urla": urla, "urlb": urlb} return doc def get_response(data): + """Sends SMS ``data`` to textit via send_sms_via_textit() function.""" - message = data.get('text') - if data.get('code') == SMS_API_ERROR: + message = data.get("text") + if data.get("code") == SMS_API_ERROR: message = None - elif data.get('code') != SMS_SUBMISSION_ACCEPTED: - message = _(u"[ERROR] %s") % message + elif data.get("code") != SMS_SUBMISSION_ACCEPTED: + message = _("[ERROR] %s") % message # send a response if message: - messages = [message, ] - sendouts = data.get('sendouts', []) + messages = [ + message, + ] + sendouts = data.get("sendouts", []) if len(sendouts): messages += sendouts for text in messages: - payload = data.get('payload', {}) - payload.update({'text': text}) - if payload.get('phone'): + payload = data.get("payload", {}) + payload.update({"text": text}) + if payload.get("phone"): send_sms_via_textit(payload) return HttpResponse() def send_sms_via_textit(payload): - response = {"phone": [payload.get('phone')], - "text": payload.get('text')} + """Returns the JsonResponse of the SMS JSOn payload for text it.""" + response = {"phone": [payload.get("phone")], "text": payload.get("text")} - return HttpResponse(json.dumps(response), content_type='application/json') + return JsonResponse(response) @require_POST @csrf_exempt def import_submission(request, username): - """ Proxy to import_submission_for_form with None as id_string """ + """Proxy to import_submission_for_form with None as id_string""" return import_submission_for_form(request, username, None) @require_POST @csrf_exempt def import_submission_for_form(request, username, id_string): - """ Retrieve and process submission from SMSSync Request """ + """Retrieve and process submission from SMSSync Request""" - sms_event = request.POST.get('event', '').strip() + sms_event = request.POST.get("event", "").strip() - if not sms_event == 'mo_sms': + if not sms_event == "mo_sms": return HttpResponse() - sms_identity = request.POST.get('phone', '').strip() - sms_relayer = request.POST.get('relayer', '').strip() - sms_text = request.POST.get('text', '').strip() + sms_identity = request.POST.get("phone", "").strip() + sms_relayer = request.POST.get("relayer", "").strip() + sms_text = request.POST.get("text", "").strip() now_time = datetime.datetime.now().isoformat() - sent_time = request.POST.get('time', now_time).strip() + sent_time = request.POST.get("time", now_time).strip() try: sms_time = dateutil.parser.parse(sent_time) except ValueError: sms_time = datetime.datetime.now() - return process_message_for_textit(username=username, - sms_identity=sms_identity, - sms_text=sms_text, - sms_time=sms_time, - id_string=id_string, - payload={'phone': sms_identity, - 'relayer': sms_relayer}) + return process_message_for_textit( + username=username, + sms_identity=sms_identity, + sms_text=sms_text, + sms_time=sms_time, + id_string=id_string, + payload={"phone": sms_identity, "relayer": sms_relayer}, + ) -def process_message_for_textit(username, sms_identity, sms_text, sms_time, - id_string, payload={}): - """ Process a text instance and return in SMSSync expected format """ +# pylint: disable=unused-argument,too-many-arguments +def process_message_for_textit( + username, sms_identity, sms_text, sms_time, id_string, payload=None +): + """Process a text instance and return in SMSSync expected format""" + payload = {} if payload is None else payload if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] - response.update({'payload': payload}) + response.update({"payload": payload}) return get_response(response) diff --git a/onadata/apps/sms_support/tests/test_base_sms.py b/onadata/apps/sms_support/tests/test_base_sms.py index 2cca175e5d..41ee0171d6 100644 --- a/onadata/apps/sms_support/tests/test_base_sms.py +++ b/onadata/apps/sms_support/tests/test_base_sms.py @@ -1,49 +1,61 @@ +# -*- coding: utf-8 -*- +""" +TestBaseSMS - base class for sms_support test cases. +""" import os import string import random -from builtins import range from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import XForm from onadata.apps.sms_support.parser import process_incoming_smses +def random_identity(): + """Returns some random digits and ascii_letters as string of length 8 used as an + identity.""" + return "".join( + [random.choice(string.digits + string.ascii_letters) for x in range(8)] + ) + + +def response_for_text(username, text, id_string=None, identity=None): + """Processes an SMS ``text`` and returns the results.""" + if identity is None: + identity = random_identity() + + return process_incoming_smses( + username=username, id_string=id_string, incomings=[(identity, text)] + )[0] + + class TestBaseSMS(TestBase): + """ + TestBaseSMS - base class for sms_support test cases. + """ def setUp(self): TestBase.setUp(self) def setup_form(self, allow_sms=True): - self.id_string = 'sms_test_form' - self.sms_keyword = 'test' - self.username = 'auser' - self.password = 'auser' + """Helper method to setup an SMS form.""" + # pylint: disable=attribute-defined-outside-init + self.id_string = "sms_test_form" + self.sms_keyword = "test" + self.username = "auser" + self.password = "auser" self.this_directory = os.path.dirname(__file__) # init FH - self._create_user_and_login(username=self.username, - password=self.password) + self._create_user_and_login(username=self.username, password=self.password) # create a test form and activate SMS Support. - self._publish_xls_file_and_set_xform(os.path.join(self.this_directory, - 'fixtures', - 'sms_tutorial.xlsx')) + self._publish_xls_file_and_set_xform( + os.path.join(self.this_directory, "fixtures", "sms_tutorial.xlsx") + ) if allow_sms: xform = XForm.objects.get(id_string=self.id_string) xform.allows_sms = True xform.save() self.xform = xform - - def random_identity(self): - return ''.join([random.choice(string.digits + string.ascii_letters) - for x in range(8)]) - - def response_for_text(self, username, text, - id_string=None, identity=None): - if identity is None: - identity = self.random_identity() - - return process_incoming_smses(username=username, - id_string=None, - incomings=[(identity, text)])[0] diff --git a/onadata/apps/sms_support/tests/test_notallowed.py b/onadata/apps/sms_support/tests/test_notallowed.py index 85039336e1..0f2399f9cf 100644 --- a/onadata/apps/sms_support/tests/test_notallowed.py +++ b/onadata/apps/sms_support/tests/test_notallowed.py @@ -1,22 +1,20 @@ from __future__ import absolute_import -from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS +from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS, response_for_text from onadata.apps.sms_support.tools import SMS_SUBMISSION_REFUSED class TestNotAllowed(TestBaseSMS): - def setUp(self): TestBaseSMS.setUp(self) self.setup_form(allow_sms=False) def test_refused_not_enabled(self): # SMS submissions not allowed - result = self.response_for_text(self.username, 'test allo') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) + result = response_for_text(self.username, "test allo") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) def test_allow_sms(self): - result = self.response_for_text(self.username, - 'test +a 1 y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) - self.assertEqual(result.get('id'), None) + result = response_for_text(self.username, "test +a 1 y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) + self.assertEqual(result.get("id"), None) diff --git a/onadata/apps/sms_support/tests/test_parser.py b/onadata/apps/sms_support/tests/test_parser.py index b85bab4c98..7ec333959a 100644 --- a/onadata/apps/sms_support/tests/test_parser.py +++ b/onadata/apps/sms_support/tests/test_parser.py @@ -1,55 +1,52 @@ from __future__ import absolute_import -from onadata.apps.sms_support.tools import (SMS_API_ERROR, SMS_PARSING_ERROR, - SMS_SUBMISSION_ACCEPTED, - SMS_SUBMISSION_REFUSED) - -from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS +from onadata.apps.sms_support.tests.test_base_sms import TestBaseSMS, response_for_text +from onadata.apps.sms_support.tools import ( + SMS_API_ERROR, + SMS_PARSING_ERROR, + SMS_SUBMISSION_ACCEPTED, + SMS_SUBMISSION_REFUSED, +) class TestParser(TestBaseSMS): - def setUp(self): TestBaseSMS.setUp(self) self.setup_form(allow_sms=True) def test_api_error(self): # missing identity or text - result = self.response_for_text(self.username, 'hello', identity='') - self.assertEqual(result['code'], SMS_API_ERROR) + result = response_for_text(self.username, "hello", identity="") + self.assertEqual(result["code"], SMS_API_ERROR) - result = self.response_for_text(self.username, text='') - self.assertEqual(result['code'], SMS_API_ERROR) + result = response_for_text(self.username, text="") + self.assertEqual(result["code"], SMS_API_ERROR) def test_invalid_syntax(self): # invalid text message - result = self.response_for_text(self.username, 'hello') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "hello") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_invalid_group(self): # invalid text message - result = self.response_for_text(self.username, '++a 20', - id_string=self.id_string) - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "++a 20", id_string=self.id_string) + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_refused_with_keyword(self): # submission has proper keywrd with invalid text - result = self.response_for_text(self.username, 'test allo') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "test allo") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_sucessful_submission(self): - result = self.response_for_text(self.username, - 'test +a 1 y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_SUBMISSION_ACCEPTED) - self.assertTrue(result['id']) + result = response_for_text(self.username, "test +a 1 y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_SUBMISSION_ACCEPTED) + self.assertTrue(result["id"]) def test_invalid_type(self): - result = self.response_for_text(self.username, - 'test +a yes y 1950-02-22 john doe') - self.assertEqual(result['code'], SMS_PARSING_ERROR) + result = response_for_text(self.username, "test +a yes y 1950-02-22 john doe") + self.assertEqual(result["code"], SMS_PARSING_ERROR) def test_missing_required_field(self): # required field name missing - result = self.response_for_text(self.username, - 'test +b ff') - self.assertEqual(result['code'], SMS_SUBMISSION_REFUSED) + result = response_for_text(self.username, "test +b ff") + self.assertEqual(result["code"], SMS_SUBMISSION_REFUSED) diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index 8e9cc5e810..033dc6cc89 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -96,7 +96,7 @@ def getsize(f): # pylint: disable=too-many-return-statements -def generate_instance(username, xml_file, media_files, uuid=None): +def generate_instance(username, xml_file, media_files, uuid=None): # noqa C901 """Process an XForm submission as if done via HTTP :param IO xml_file: file-like object containing XML XForm @@ -154,7 +154,7 @@ def generate_instance(username, xml_file, media_files, uuid=None): } -def is_sms_related(json_survey): +def is_sms_related(json_survey): # noqa C901 """Whether a form is considered to want sms Support return True if one sms-related field is defined.""" @@ -183,8 +183,8 @@ def _walk(dl): return _walk(json_survey) -# pylint: disable=too-many-locals,too-many-branches -def check_form_sms_compatibility(form, json_survey=None): +# pylint: disable=too-many-locals,too-many-branches,too-many-statements +def check_form_sms_compatibility(form, json_survey=None): # noqa C901 """Tests all SMS related rules on the XForm representation Returns a view-compatible dict(type, text) with warnings or diff --git a/onadata/apps/sms_support/views.py b/onadata/apps/sms_support/views.py index f8c1a82eb3..7d4f43e695 100644 --- a/onadata/apps/sms_support/views.py +++ b/onadata/apps/sms_support/views.py @@ -1,11 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 nu +""" +sms_support views. +""" from __future__ import absolute_import import json -from django.http import HttpResponse +from django.http import JsonResponse from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_POST @@ -15,16 +18,20 @@ def get_response(data): - response = {'status': data.get('code'), - 'message': data.get('text'), - 'instanceID': data.get('id'), - 'sendouts': data.get('sendouts')} - return HttpResponse(json.dumps(response), content_type='application/json') + """Returns a JsonResponse object with `status`, `message`, `instanceID` and + `sendouts` based on the input ``data`` object.""" + response = { + "status": data.get("code"), + "message": data.get("text"), + "instanceID": data.get("id"), + "sendouts": data.get("sendouts"), + } + return JsonResponse(response) @require_GET def import_submission(request, username): - """ Process an SMS text as a form submission + """Process an SMS text as a form submission :param string identity: phone number of the sender :param string text: SMS content @@ -34,7 +41,7 @@ def import_submission(request, username): 'message': Error message if not ACCEPTED. 'id: Unique submission ID if ACCEPTED. - """ + """ return import_submission_for_form(request, username, None) @@ -42,46 +49,57 @@ def import_submission(request, username): @require_POST @csrf_exempt def import_multiple_submissions(request, username): - ''' Process several POSTED SMS texts as XForm submissions + """Process several POSTED SMS texts as XForm submissions :param json messages: JSON list of {"identity": "x", "text": "x"} :returns json list of {"status": "x", "message": "x", "id": "x"} - ''' + """ return import_multiple_submissions_for_form(request, username, None) @require_GET def import_submission_for_form(request, username, id_string): - """ idem import_submission with a defined id_string """ + """idem import_submission with a defined id_string""" - sms_identity = request.GET.get('identity', '').strip() - sms_text = request.GET.get('text', '').strip() + sms_identity = request.GET.get("identity", "").strip() + sms_text = request.GET.get("text", "").strip() if not sms_identity or not sms_text: - return get_response({'code': SMS_API_ERROR, - 'text': _(u"`identity` and `message` are " - u"both required and must not be " - u"empty.")}) + return get_response( + { + "code": SMS_API_ERROR, + "text": _( + "`identity` and `message` are " + "both required and must not be " + "empty." + ), + } + ) incomings = [(sms_identity, sms_text)] response = process_incoming_smses(username, incomings, id_string)[-1] return get_response(response) +# pylint: disable=invalid-name @require_POST @csrf_exempt def import_multiple_submissions_for_form(request, username, id_string): - """ idem import_multiple_submissions with a defined id_string """ - - messages = json.loads(request.POST.get('messages', '[]')) - incomings = [(m.get('identity', ''), m.get('text', '')) for m in messages] - - responses = [{'status': d.get('code'), - 'message': d.get('text'), - 'instanceID': d.get('id'), - 'sendouts': d.get('sendouts')} for d - in process_incoming_smses(username, incomings, id_string)] - - return HttpResponse(json.dumps(responses), content_type='application/json') + """idem import_multiple_submissions with a defined id_string""" + + messages = json.loads(request.POST.get("messages", "[]")) + incomings = [(m.get("identity", ""), m.get("text", "")) for m in messages] + + responses = [ + { + "status": d.get("code"), + "message": d.get("text"), + "instanceID": d.get("id"), + "sendouts": d.get("sendouts"), + } + for d in process_incoming_smses(username, incomings, id_string) + ] + + return JsonResponse(responses, safe=False) From 4e6093b737b2d23b01c103f669350bed2207277f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 19:48:44 +0300 Subject: [PATCH 201/234] logger_tools.py: cleanup --- onadata/apps/logger/models/xform.py | 48 ++--- .../logger/tests/test_simple_submission.py | 2 +- onadata/apps/main/models/meta_data.py | 15 +- .../management/commands/mark_start_times.py | 2 +- .../management/commands/set_uuid_in_xml.py | 2 +- onadata/apps/viewer/models/data_dictionary.py | 6 +- onadata/libs/utils/logger_tools.py | 178 +++++++++++------- 7 files changed, 144 insertions(+), 109 deletions(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 0f43ebba1e..a605382ef5 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -10,12 +10,11 @@ from datetime import datetime from xml.dom import Node -import pytz from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.contenttypes.fields import GenericRelation -from django.core.exceptions import ObjectDoesNotExist from django.core.cache import cache +from django.core.exceptions import ObjectDoesNotExist from django.db import models, transaction from django.db.models import Sum from django.db.models.signals import post_delete, pre_save @@ -24,12 +23,14 @@ from django.utils.html import conditional_escape from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy -from six import iteritems + +import pytz from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question from pyxform.section import RepeatingSection from pyxform.xform2json import create_survey_element_from_xml +from six import iteritems from taggit.managers import TaggableManager from onadata.apps.logger.xform_instance_parser import XLSFormError, clean_and_parse_xml @@ -38,30 +39,30 @@ PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, PROJ_SUB_DATE_CACHE, XFORM_COUNT, - PROJ_OWNER_CACHE, XFORM_SUBMISSION_COUNT_FOR_DAY, XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete, ) from onadata.libs.utils.common_tags import ( + DATE_MODIFIED, DURATION, ID, KNOWN_MEDIA_TYPES, MEDIA_ALL_RECEIVED, MEDIA_COUNT, + MULTIPLE_SELECT_TYPE, NOTES, + REVIEW_COMMENT, + REVIEW_STATUS, SUBMISSION_TIME, SUBMITTED_BY, TAGS, TOTAL_MEDIA, UUID, VERSION, - REVIEW_STATUS, - REVIEW_COMMENT, - MULTIPLE_SELECT_TYPE, - DATE_MODIFIED, ) from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.utils.mongo import _encode_for_mongo @@ -234,7 +235,7 @@ class XFormMixin: PREFIX_NAME_REGEX = re.compile(r"(?P.+/)(?P[^/]+)$") # pylint: disable=too-many-locals - def _set_uuid_in_xml(self, file_name=None): + def set_uuid_in_xml(self, file_name=None): """ Add bind to automatically set UUID node in XML. """ @@ -313,13 +314,13 @@ def _set_uuid_in_xml(self, file_name=None): # hack # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ - text_re = re.compile("(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*") pretty_xml = text_re.sub( lambda m: "".join(m.group(1, 2, 3)), self.xml.decode("utf-8") ) inline_output = output_re.sub("\g<1>", pretty_xml) # noqa - inline_output = re.compile("").sub( # noqa + inline_output = re.compile(r"").sub( "", inline_output ) self.xml = inline_output @@ -718,7 +719,8 @@ def get_data_for_excel(self): self._expand_geocodes(row, key, elem) yield row - def _mark_start_time_boolean(self): + def mark_start_time_boolean(self): + """Sets True the `self.has_start_time` if the form has a start meta question.""" starttime_substring = 'jr:preloadParams="start"' if self.xml.find(starttime_substring) != -1: self.has_start_time = True @@ -726,6 +728,7 @@ def _mark_start_time_boolean(self): self.has_start_time = False def get_survey_elements_of_type(self, element_type): + """Returns all survey elements of type ``element_type``.""" return [e for e in self.get_survey_elements() if e.type == element_type] # pylint: disable=invalid-name @@ -932,7 +935,7 @@ def _set_title(self): if isinstance(self.xml, bytes): self.xml = self.xml.decode("utf-8") self.xml = TITLE_PATTERN.sub(f"{title_xml}", self.xml) - self._set_hash() + self.set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( _("Title shouldn't have any invalid xml " "characters ('>' '&' '<')") @@ -947,7 +950,15 @@ def _set_title(self): self.title = title_xml - def _set_hash(self): + def get_hash(self): + """Returns the MD5 hash of the forms XML content prefixed by 'md5:'""" + md5_hash = hashlib.new( + "md5", self.xml.encode("utf-8"), usedforsecurity=False + ).hexdigest() + return f"md5:{md5_hash}" + + def set_hash(self): + """Sets the MD5 hash of the form.""" self.hash = self.get_hash() def _set_encrypted_field(self): @@ -985,7 +996,7 @@ def save(self, *args, **kwargs): # noqa: MC0001 if update_fields is None or "title" in update_fields: self._set_title() if self.pk is None: - self._set_hash() + self.set_hash() if update_fields is None or "encrypted" in update_fields: self._set_encrypted_field() if update_fields is None or "id_string" in update_fields: @@ -1174,13 +1185,6 @@ def time_of_last_submission_update(self): return last_submission_time - def get_hash(self): - """Returns the MD5 hash of the forms XML content prefixed by 'md5:'""" - md5_hash = hashlib.new( - "md5", self.xml.encode("utf-8"), usedforsecurity=False - ).hexdigest() - return f"md5:{md5_hash}" - @property def can_be_replaced(self): """Returns True if the form has zero submissions - forms with zero permissions diff --git a/onadata/apps/logger/tests/test_simple_submission.py b/onadata/apps/logger/tests/test_simple_submission.py index b045eff26e..b8ab174981 100644 --- a/onadata/apps/logger/tests/test_simple_submission.py +++ b/onadata/apps/logger/tests/test_simple_submission.py @@ -33,7 +33,7 @@ def _get_xml_for_form(self, xform): builder = SurveyElementBuilder() sss = builder.create_survey_element_from_json(xform.json) xform.xml = sss.to_xml() - xform._mark_start_time_boolean() + xform.mark_start_time_boolean() xform.save() def _submit_at_hour(self, hour): diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index e94094b2e8..025af48010 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -182,6 +182,8 @@ def media_resources(media_list, download=False): # pylint: disable=too-many-public-methods class MetaData(models.Model): + """MetaData class model.""" + data_type = models.CharField(max_length=255) data_value = models.CharField(max_length=255) data_file = models.FileField(upload_to=upload_to, blank=True, null=True) @@ -204,7 +206,7 @@ class Meta: # pylint: disable=arguments-differ def save(self, *args, **kwargs): - self._set_hash() + self.set_hash() super().save(*args, **kwargs) @property @@ -215,15 +217,9 @@ def hash(self): if self.file_hash is not None and self.file_hash != "": return self.file_hash - return self._set_hash() + return self.set_hash() def set_hash(self): - """ - Returns the md5 hash of the metadata file. - """ - return self._set_hash() - - def _set_hash(self): """ Returns the md5 hash of the metadata file. """ @@ -251,7 +247,6 @@ def _set_hash(self): def soft_delete(self): """ - Return the soft deletion timestamp Mark the MetaData as soft deleted, by updating the deleted_at field. """ @@ -261,6 +256,7 @@ def soft_delete(self): @staticmethod def public_link(content_object, data_value=None): + """Returns the public link metadata.""" data_type = "public_link" if data_value is False: data_value = "False" @@ -270,6 +266,7 @@ def public_link(content_object, data_value=None): @staticmethod def set_google_sheet_details(content_object, data_value=None): + """Returns Google Sheet details metadata object.""" data_type = GOOGLE_SHEET_DATA_TYPE return unique_type_for_form(content_object, data_type, data_value) diff --git a/onadata/apps/viewer/management/commands/mark_start_times.py b/onadata/apps/viewer/management/commands/mark_start_times.py index b6c4cd7286..096fab9b39 100644 --- a/onadata/apps/viewer/management/commands/mark_start_times.py +++ b/onadata/apps/viewer/management/commands/mark_start_times.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): for dd in DataDictionary.objects.all(): try: - dd._mark_start_time_boolean() + dd.mark_start_time_boolean() dd.save() except Exception: self.stderr.write( diff --git a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py index 86c31b6fcd..0faf46ed7b 100644 --- a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py +++ b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py @@ -14,7 +14,7 @@ def handle(self, *args, **kwargs): for i, dd in enumerate( queryset_iterator(DataDictionary.objects.all())): if dd.xls: - dd._set_uuid_in_xml() + dd.set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i}) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index d73de5c7c4..2b77106fb4 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -192,10 +192,10 @@ def save(self, *args, **kwargs): self.version = survey.get("version") self.last_updated_at = timezone.now() self.title = survey.get("title") - self._mark_start_time_boolean() + self.mark_start_time_boolean() set_uuid(self) - self._set_uuid_in_xml() - self._set_hash() + self.set_uuid_in_xml() + self.set_hash() if "skip_xls_read" in kwargs: del kwargs["skip_xls_read"] diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 9361cc7300..c20b8b5992 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +logger_tools - Logger app utility functions. +""" import json import os import re @@ -10,20 +14,17 @@ from typing import NoReturn from wsgiref.util import FileWrapper from xml.dom import Node -import xml.etree.ElementTree as ET from xml.parsers.expat import ExpatError -import pytz -from dict2xml import dict2xml from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.exceptions import ( MultipleObjectsReturned, PermissionDenied, ValidationError, ) from django.core.files.storage import get_storage_class -from django.db import IntegrityError, transaction, DataError +from django.db import DataError, IntegrityError, transaction from django.db.models import Q from django.http import ( HttpResponse, @@ -35,6 +36,10 @@ from django.utils import timezone from django.utils.encoding import DjangoUnicodeDecodeError from django.utils.translation import gettext as _ + +import pytz +from defusedxml.ElementTree import fromstring +from dict2xml import dict2xml from modilabs.utils.subprocess_timeout import ProcessTimedOut from multidb.pinning import use_master from pyxform.errors import PyXFormError @@ -44,29 +49,29 @@ from onadata.apps.logger.models import Attachment, Instance, XForm, XFormVersion from onadata.apps.logger.models.instance import ( FormInactiveError, - InstanceHistory, FormIsMergedDatasetError, + InstanceHistory, get_id_string_from_xml_str, ) from onadata.apps.logger.models.xform import XLSFormError from onadata.apps.logger.xform_instance_parser import ( + AttachmentNameError, DuplicateInstance, InstanceEmptyError, + InstanceEncryptionError, + InstanceFormatError, InstanceInvalidUserError, InstanceMultipleNodeError, - InstanceEncryptionError, NonUniqueFormIdError, - InstanceFormatError, clean_and_parse_xml, get_deprecated_uuid_from_xml, get_submission_date_from_xml, get_uuid_from_xml, - AttachmentNameError, ) from onadata.apps.messaging.constants import ( - XFORM, - SUBMISSION_EDITED, SUBMISSION_CREATED, + SUBMISSION_EDITED, + XFORM, ) from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.data_dictionary import DataDictionary @@ -74,7 +79,7 @@ from onadata.apps.viewer.signals import process_submission from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.common_tags import METADATA_FIELDS -from onadata.libs.utils.common_tools import report_exception, get_uuid +from onadata.libs.utils.common_tools import get_uuid, report_exception from onadata.libs.utils.model_tools import set_uuid from onadata.libs.utils.user_auth import get_user_default_project @@ -95,13 +100,18 @@ ) +# pylint: disable=invalid-name +User = get_user_model() + + def create_xform_version(xform: XForm, user: User) -> XFormVersion: """ Creates an XFormVersion object for the passed in XForm """ + versioned_xform = None try: with transaction.atomic(): - return XFormVersion.objects.create( + versioned_xform = XFormVersion.objects.create( xform=xform, xls=xform.xls, json=xform.json @@ -113,8 +123,10 @@ def create_xform_version(xform: XForm, user: User) -> XFormVersion: ) except IntegrityError: pass + return versioned_xform +# pylint: disable=too-many-arguments def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum, request=None): history = None instance = None @@ -208,16 +220,14 @@ def dict2xform(jsform, form_id, root=None, username=None, gen_uuid=False): if gen_uuid: jsform["meta"] = {"instanceID": "uuid:" + get_uuid(hex_only=False)} - return "<{0} id='{1}'>{2}".format( - root, form_id, dict2xml(jsform) - ) + return f"<{root} id='{form_id}'>{dict2xml(jsform)}" def get_first_record(queryset): """ Returns the first item in a queryset sorted by id. """ - records = sorted([record for record in queryset], key=lambda k: k.id) + records = sorted(list(queryset), key=lambda k: k.id) if records: return records[0] @@ -225,11 +235,12 @@ def get_first_record(queryset): def get_uuid_from_submission(xml): + """Extracts and returns the UUID from a submission XML.""" # parse UUID from uploaded XML split_xml = uuid_regex.split(xml.decode("utf-8")) # check that xml has UUID - return len(split_xml) > 1 and split_xml[1] or None + return split_xml[1] if len(split_xml) > 1 else None def get_xform_from_submission(xml, username, uuid=None, request=None): @@ -268,7 +279,7 @@ def get_xform_from_submission(xml, username, uuid=None, request=None): # Assumption: If the owner_username is equal to the XForm # owner we've retrieved the correct form. if username and xform.user.username == username: - raise e + raise e from e else: return xform @@ -280,8 +291,8 @@ def get_xform_from_submission(xml, username, uuid=None, request=None): user__username__iexact=username, deleted_at__isnull=True, ) - except MultipleObjectsReturned: - raise NonUniqueFormIdError() + except MultipleObjectsReturned as e: + raise NonUniqueFormIdError() from e def _has_edit_xform_permission(xform, user): @@ -292,6 +303,7 @@ def _has_edit_xform_permission(xform, user): def check_edit_submission_permissions(request_user, xform): + """Checks edit submission permissions.""" if xform and request_user and request_user.is_authenticated: requires_auth = xform.user.profile.require_auth has_edit_perms = _has_edit_xform_permission(xform, request_user) @@ -324,13 +336,14 @@ def check_submission_permissions(request, xform): :returns: None. :raises: PermissionDenied based on the above criteria. """ + requires_authentication = ( + xform.user.profile.require_auth + or xform.require_auth + or request.path == "/submission" + ) if ( request - and ( - xform.user.profile.require_auth - or xform.require_auth - or request.path == "/submission" - ) + and requires_authentication and xform.user != request.user and not request.user.has_perm("report_xform", xform) ): @@ -356,7 +369,7 @@ def check_submission_encryption(xform: XForm, xml: bytes) -> NoReturn: from the submissions """ submission_encrypted = False - submission_element = ET.fromstring(xml) + submission_element = fromstring(xml) encrypted_attrib = submission_element.attrib.get("encrypted") required_encryption_elems = [ elem.tag @@ -438,6 +451,7 @@ def save_submission( checksum, request=None, ): + """Persist a submission into the ParsedInstance model.""" if not date_created_override: date_created_override = get_submission_date_from_xml(xml) @@ -471,6 +485,7 @@ def get_filtered_instances(*args, **kwargs): return Instance.objects.filter(*args, **kwargs) +# pylint: disable=too-many-locals def create_instance( username, xml_file, @@ -577,8 +592,9 @@ def create_instance( return instance +# pylint: disable=too-many-branches,too-many-statements @use_master -def safe_create_instance( +def safe_create_instance( # noqa C901 username, xml_file, media_files, @@ -633,9 +649,7 @@ def safe_create_instance( except PermissionDenied as e: error = OpenRosaResponseForbidden(e) except UnreadablePostError as e: - error = OpenRosaResponseBadRequest( - _("Unable to read submitted file: %(error)s" % {"error": text(e)}) - ) + error = OpenRosaResponseBadRequest(_(f"Unable to read submitted file: {e}")) except InstanceMultipleNodeError as e: error = OpenRosaResponseBadRequest(e) except DjangoUnicodeDecodeError: @@ -667,10 +681,13 @@ def response_with_mimetype_and_name( use_local_filesystem=False, full_mime=False, ): + """Returns a HttpResponse with Content-Disposition header set + + Triggers a download on the browser.""" if extension is None: extension = mimetype if not full_mime: - mimetype = "application/%s" % mimetype + mimetype = f"application/{mimetype}" if file_path: try: if not use_local_filesystem: @@ -679,7 +696,8 @@ def response_with_mimetype_and_name( response = StreamingHttpResponse(wrapper, content_type=mimetype) response["Content-Length"] = default_storage.size(file_path) else: - wrapper = FileWrapper(open(file_path)) + # pylint: disable=consider-using-with + wrapper = FileWrapper(open(file_path, "rb")) response = StreamingHttpResponse(wrapper, content_type=mimetype) response["Content-Length"] = os.path.getsize(file_path) except IOError: @@ -693,22 +711,23 @@ def response_with_mimetype_and_name( def generate_content_disposition_header(name, extension, show_date=True): + """Returns the a Content-Description header formatting string,""" if name is None: return "attachment;" if show_date: - name = "%s-%s" % (name, datetime.now().strftime("%Y-%m-%d-%H-%M-%S")) - return "attachment; filename=%s.%s" % (name, extension) + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + name = f"{name}-{timestamp}" + return f"attachment; filename={name}.{extension}" def store_temp_file(data): - tmp = tempfile.TemporaryFile() + """Creates a temporary file with the ``data`` and returns it.""" ret = None - try: + with tempfile.TemporaryFile() as tmp: tmp.write(data) tmp.seek(0) ret = tmp - finally: - tmp.close() + return ret @@ -739,10 +758,8 @@ def publish_form(callback): ("An error occurred while publishing the form. " "Please try again.") ), } - except (AttributeError, Exception, ValidationError) as e: - report_exception( - "Form publishing exception: {}".format(e), text(e), sys.exc_info() - ) + except (AttributeError, ValidationError) as e: + report_exception(f"Form publishing exception: {e}", text(e), sys.exc_info()) return {"type": "alert-error", "text": text(e)} @@ -777,6 +794,7 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): additional_context={"from": "Publish XML Form"}, ) def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): + """Publish an XML XForm.""" xml = xml_file.read() if isinstance(xml, bytes): xml = xml.decode("utf-8") @@ -786,20 +804,20 @@ def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): dd = DataDictionary.objects.get(user=user, id_string=id_string, project=project) dd.xml = xml dd.json = form_json - dd._mark_start_time_boolean() + dd.mark_start_time_boolean() set_uuid(dd) - dd._set_uuid_in_xml() - dd._set_hash() + dd.set_uuid_in_xml() + dd.set_hash() dd.save() else: created_by = created_by or user dd = DataDictionary( created_by=created_by, user=user, xml=xml, json=form_json, project=project ) - dd._mark_start_time_boolean() + dd.mark_start_time_boolean() set_uuid(dd) - dd._set_uuid_in_xml(file_name=xml_file.name) - dd._set_hash() + dd.set_uuid_in_xml(file_name=xml_file.name) + dd.set_hash() dd.save() # Create an XFormVersion object for the published XLSForm @@ -820,63 +838,75 @@ def remove_metadata_fields(data): return data +def set_default_openrosa_headers(response): + """Sets the default OpenRosa headers into a ``response`` object.""" + response["Content-Type"] = "text/html; charset=utf-8" + response["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH + tz = pytz.timezone(settings.TIME_ZONE) + dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") + response["Date"] = dt + response[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION + response["Content-Type"] = DEFAULT_CONTENT_TYPE + + class BaseOpenRosaResponse(HttpResponse): + """The base HTTP response class with OpenRosa headers.""" + status_code = 201 def __init__(self, *args, **kwargs): - super(BaseOpenRosaResponse, self).__init__(*args, **kwargs) - - self[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") - self["Date"] = dt - self["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH - self["Content-Type"] = DEFAULT_CONTENT_TYPE + super().__init__(*args, **kwargs) + set_default_openrosa_headers(self) class OpenRosaResponse(BaseOpenRosaResponse): + """An HTTP response class with OpenRosa headers for the created response.""" + status_code = 201 def __init__(self, *args, **kwargs): - super(OpenRosaResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.message = self.content # wrap content around xml - self.content = ( - """ + self.content = f""" - %s + {self.content} """ - % self.content - ) class OpenRosaResponseNotFound(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Not Found response.""" + status_code = 404 class OpenRosaResponseBadRequest(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Bad Request response.""" + status_code = 400 class OpenRosaResponseNotAllowed(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Not Allowed response.""" + status_code = 405 class OpenRosaResponseForbidden(OpenRosaResponse): + """An HTTP response class with OpenRosa headers for the Forbidden response.""" + status_code = 403 class OpenRosaNotAuthenticated(Response): + """An HTTP response class with OpenRosa headers for the Not Authenticated + response.""" + status_code = 401 def __init__(self, *args, **kwargs): - super(OpenRosaNotAuthenticated, self).__init__(*args, **kwargs) - - self["Content-Type"] = "text/html; charset=utf-8" - self["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") - self["Date"] = dt + super().__init__(*args, **kwargs) + set_default_openrosa_headers(self) def inject_instanceid(xml_str, uuid): @@ -911,22 +941,26 @@ def inject_instanceid(xml_str, uuid): else: uuid_tag = uuid_tags[0] # insert meta and instanceID - text_node = xml.createTextNode("uuid:%s" % uuid) + text_node = xml.createTextNode(f"uuid:{uuid}") uuid_tag.appendChild(text_node) return xml.toxml() return xml_str def remove_xform(xform): + """Deletes an XForm ``xform``.""" # delete xform, and all related models xform.delete() -class PublishXForm(object): +class PublishXForm: + "A class to publish an XML XForm file." + def __init__(self, xml_file, user): self.xml_file = xml_file self.user = user self.project = get_user_default_project(user) def publish_xform(self): + """Publish an XForm XML file.""" return publish_xml_form(self.xml_file, self.user, self.project) From e7f68820de45753c8625c42b5ca6e58c2848bc6e Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 21:35:42 +0300 Subject: [PATCH 202/234] logger_tools.py: Fix check for request. --- onadata/apps/sms_support/parser.py | 4 ++-- onadata/apps/sms_support/tests/test_base_sms.py | 2 +- onadata/libs/utils/logger_tools.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index 44ee14f598..05ed039ee1 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -47,7 +47,7 @@ def __init__(self, message, question=None): super().__init__(message) -# pylint: disable=too-many-locals,too-many-branches +# pylint: disable=too-many-locals,too-many-branches,too-many-statements def parse_sms_text(xform, identity, sms_text): # noqa C901 """Parses an SMS text to return XForm specific answers, media, notes.""" @@ -216,7 +216,7 @@ def get_meta_value(xlsf_type, identity): real_value = None question_type = question.get("type") - if question_type in ("calculate"): + if question_type == "calculate": # 'calculate' question are not implemented. # 'note' ones are just meant to be displayed on device continue diff --git a/onadata/apps/sms_support/tests/test_base_sms.py b/onadata/apps/sms_support/tests/test_base_sms.py index 41ee0171d6..eb4dcd53f5 100644 --- a/onadata/apps/sms_support/tests/test_base_sms.py +++ b/onadata/apps/sms_support/tests/test_base_sms.py @@ -15,7 +15,7 @@ def random_identity(): """Returns some random digits and ascii_letters as string of length 8 used as an identity.""" return "".join( - [random.choice(string.digits + string.ascii_letters) for x in range(8)] + [random.choice(string.digits + string.ascii_letters) for x in range(8)] # nosec ) diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index c20b8b5992..f200409ad4 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -336,14 +336,13 @@ def check_submission_permissions(request, xform): :returns: None. :raises: PermissionDenied based on the above criteria. """ - requires_authentication = ( + requires_authentication = request and ( xform.user.profile.require_auth or xform.require_auth or request.path == "/submission" ) if ( - request - and requires_authentication + requires_authentication and xform.user != request.user and not request.user.has_perm("report_xform", xform) ): From 4a2fa71f7920f5f304b055b9c1be9a169dcf6d36 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 11 May 2022 23:01:57 +0300 Subject: [PATCH 203/234] logger_tools.py: Fix add DuplicateUUIDError exception for handling in publishing a form. --- onadata/libs/utils/logger_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index f200409ad4..85e4a588ed 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -53,7 +53,7 @@ InstanceHistory, get_id_string_from_xml_str, ) -from onadata.apps.logger.models.xform import XLSFormError +from onadata.apps.logger.models.xform import DuplicateUUIDError, XLSFormError from onadata.apps.logger.xform_instance_parser import ( AttachmentNameError, DuplicateInstance, @@ -757,7 +757,7 @@ def publish_form(callback): ("An error occurred while publishing the form. " "Please try again.") ), } - except (AttributeError, ValidationError) as e: + except (AttributeError, DuplicateUUIDError, ValidationError) as e: report_exception(f"Form publishing exception: {e}", text(e), sys.exc_info()) return {"type": "alert-error", "text": text(e)} From 3ee8df8828cab1b0cb268e84e3055af69cee2698 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 9 May 2022 20:47:59 +0300 Subject: [PATCH 204/234] apps/logger blackify --- onadata/apps/logger/admin.py | 12 +- onadata/apps/logger/import_tools.py | 56 +- .../apps/logger/management/commands/add_id.py | 27 +- .../management/commands/create_backup.py | 14 +- .../management/commands/export_gps_points.py | 12 +- .../commands/fix_attachments_counts.py | 28 +- .../commands/fix_duplicate_instances.py | 28 +- .../commands/fix_submission_count.py | 27 +- .../commands/generate_platform_stats.py | 48 +- .../apps/logger/management/commands/import.py | 4 +- .../management/commands/import_tools.py | 31 +- .../commands/pull_from_aggregate.py | 25 +- .../commands/recover_deleted_attachments.py | 22 +- .../remove_columns_from_briefcase_data.py | 79 +- .../commands/replace_form_id_root_node.py | 46 +- .../commands/set_xform_surveys_with_osm.py | 11 +- .../management/commands/transferproject.py | 65 +- .../management/commands/update_xform_uuids.py | 34 +- .../migrations/0001_pre-django-3-upgrade.py | 1336 +++++++++++++---- .../migrations/0002_auto_20150717_0048.py | 14 +- .../migrations/0002_auto_20220425_0340.py | 44 +- .../0003_alter_instance_media_all_received.py | 10 +- .../0003_dataview_instances_with_geopoints.py | 6 +- .../migrations/0004_auto_20150910_0056.py | 77 +- .../migrations/0005_auto_20151015_0758.py | 77 +- .../migrations/0007_osmdata_field_name.py | 8 +- .../migrations/0008_osmdata_osm_type.py | 8 +- .../migrations/0009_auto_20151111_0438.py | 6 +- .../migrations/0010_attachment_file_size.py | 6 +- .../0011_dataview_matches_parent.py | 6 +- .../migrations/0012_auto_20160114_0708.py | 8 +- .../logger/migrations/0013_note_created_by.py | 14 +- .../migrations/0014_note_instance_field.py | 6 +- .../migrations/0015_auto_20160222_0559.py | 13 +- .../migrations/0016_widget_aggregation.py | 9 +- .../migrations/0017_auto_20160224_0130.py | 23 +- .../migrations/0018_auto_20160301_0330.py | 17 +- .../migrations/0020_auto_20160408_0325.py | 36 +- .../migrations/0021_auto_20160408_0919.py | 6 +- .../migrations/0022_auto_20160418_0518.py | 10 +- .../migrations/0023_auto_20160419_0403.py | 13 +- .../migrations/0024_xform_has_hxl_support.py | 6 +- .../migrations/0025_xform_last_updated_at.py | 10 +- .../migrations/0026_auto_20160913_0239.py | 6 +- .../migrations/0027_auto_20161201_0730.py | 9 +- .../migrations/0028_auto_20170217_0502.py | 63 +- .../migrations/0028_auto_20170221_0838.py | 40 +- .../migrations/0029_auto_20170221_0908.py | 11 +- .../migrations/0030_auto_20170227_0137.py | 10 +- onadata/apps/logger/migrations/0031_merge.py | 7 +- .../migrations/0032_project_deleted_at.py | 6 +- .../migrations/0033_auto_20170705_0159.py | 34 +- .../migrations/0034_auto_20170814_0432.py | 23 +- .../logger/migrations/0034_mergedxform.py | 35 +- .../migrations/0035_auto_20170712_0529.py | 12 +- .../0036_xform_is_merged_dataset.py | 6 +- .../migrations/0037_merge_20170825_0238.py | 7 +- .../migrations/0038_auto_20170828_1718.py | 27 +- .../migrations/0039_auto_20170909_2052.py | 10 +- .../migrations/0040_auto_20170912_1504.py | 10 +- .../migrations/0041_auto_20170912_1512.py | 8 +- .../apps/logger/migrations/0042_xform_hash.py | 11 +- .../migrations/0043_auto_20171010_0403.py | 11 +- .../migrations/0044_xform_hash_sql_update.py | 4 +- .../logger/migrations/0045_attachment_name.py | 6 +- .../migrations/0046_auto_20180314_1618.py | 8 +- .../migrations/0047_dataview_deleted_at.py | 6 +- .../migrations/0048_dataview_deleted_by.py | 11 +- .../migrations/0049_xform_deleted_by.py | 10 +- .../migrations/0050_project_deleted_by.py | 15 +- .../migrations/0051_auto_20180522_1118.py | 17 +- .../migrations/0052_auto_20180805_2233.py | 11 +- .../migrations/0053_submissionreview.py | 111 +- .../migrations/0054_instance_has_a_review.py | 8 +- .../migrations/0055_auto_20180904_0713.py | 6 +- .../migrations/0056_auto_20190125_0517.py | 131 +- .../migrations/0057_xform_public_key.py | 8 +- .../migrations/0058_auto_20191211_0900.py | 8 +- .../migrations/0059_attachment_deleted_by.py | 13 +- .../migrations/0060_auto_20200305_0357.py | 8 +- .../migrations/0061_auto_20200713_0814.py | 9 +- .../migrations/0062_auto_20210202_0248.py | 13 +- .../logger/migrations/0063_xformversion.py | 46 +- .../migrations/0064_auto_20210304_0314.py | 11 +- onadata/apps/logger/models/attachment.py | 50 +- onadata/apps/logger/models/note.py | 14 +- .../apps/logger/models/submission_review.py | 47 +- onadata/apps/logger/models/xform_version.py | 10 +- .../test_recover_deleted_attachments.py | 38 +- ...test_remove_columns_from_briefcase_data.py | 16 +- .../test_replace_form_id_root_node.py | 31 +- .../logger/tests/models/test_attachment.py | 107 +- .../logger/tests/models/test_data_view.py | 168 +-- .../apps/logger/tests/models/test_instance.py | 176 ++- onadata/apps/logger/tests/models/test_note.py | 4 +- .../tests/models/test_submission_review.py | 2 +- .../apps/logger/tests/test_backup_tools.py | 63 +- .../apps/logger/tests/test_briefcase_api.py | 221 +-- .../tests/test_digest_authentication.py | 90 +- .../tests/test_encrypted_submissions.py | 82 +- onadata/apps/logger/tests/test_form_list.py | 17 +- .../logger/tests/test_importing_database.py | 51 +- .../logger/tests/test_instance_creation.py | 58 +- onadata/apps/logger/tests/test_parsing.py | 188 ++- onadata/apps/logger/tests/test_publish_xls.py | 54 +- .../logger/tests/test_simple_submission.py | 66 +- .../logger/tests/test_update_xform_uuid.py | 6 +- onadata/apps/logger/tests/test_webforms.py | 52 +- onadata/apps/logger/xform_fs.py | 23 +- 109 files changed, 2944 insertions(+), 1834 deletions(-) diff --git a/onadata/apps/logger/admin.py b/onadata/apps/logger/admin.py index 5366ea069d..a62cbc869f 100644 --- a/onadata/apps/logger/admin.py +++ b/onadata/apps/logger/admin.py @@ -6,9 +6,9 @@ class XFormAdmin(VersionAdmin, admin.ModelAdmin): - exclude = ('user',) - list_display = ('id_string', 'downloadable', 'shared') - search_fields = ('id_string', 'title') + exclude = ("user",) + list_display = ("id_string", "downloadable", "shared") + search_fields = ("id_string", "title") # A user should only see forms that belong to him. def get_queryset(self, request): @@ -23,9 +23,9 @@ def get_queryset(self, request): class ProjectAdmin(VersionAdmin, admin.ModelAdmin): list_max_show_all = 2000 - list_select_related = ('organization',) - ordering = ['name'] - search_fields = ('name', 'organization__username', 'organization__email') + list_select_related = ("organization",) + ordering = ["name"] + search_fields = ("name", "organization__username", "organization__email") # A user should only see projects that belong to him. def get_queryset(self, request): diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index 4b0b017f95..ae1d861a46 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -33,30 +33,31 @@ def django_file(path, field_name, content_type): # adapted from here: # http://groups.google.com/group/django-users/browse_thread/thread/ # 834f988876ff3c45/ - f = open(path, 'rb') + f = open(path, "rb") return InMemoryUploadedFile( file=f, field_name=field_name, name=f.name, content_type=content_type, size=os.path.getsize(path), - charset=None + charset=None, ) -def import_instance(username, xform_path, photos, osm_files, status, - raise_exception): +def import_instance(username, xform_path, photos, osm_files, status, raise_exception): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ - with django_file(xform_path, field_name="xml_file", - content_type="text/xml") as xml_file: - images = [django_file(jpg, field_name="image", - content_type="image/jpeg") for jpg in photos] + with django_file( + xform_path, field_name="xml_file", content_type="text/xml" + ) as xml_file: + images = [ + django_file(jpg, field_name="image", content_type="image/jpeg") + for jpg in photos + ] images += [ - django_file(osm, field_name='image', - content_type='text/xml') + django_file(osm, field_name="image", content_type="text/xml") for osm in osm_files ] try: @@ -79,8 +80,9 @@ def import_instance_async(username, xform_path, photos, osm_files, status): import_instance(username, xform_path, photos, osm_files, status, False) -def iterate_through_instances(dirpath, callback, user=None, status='zip', - is_async=False): +def iterate_through_instances( + dirpath, callback, user=None, status="zip", is_async=False +): total_file_count = 0 success_count = 0 errors = [] @@ -91,16 +93,17 @@ def iterate_through_instances(dirpath, callback, user=None, status='zip', if XFormInstanceFS.is_valid_instance(filepath): xfxs = XFormInstanceFS(filepath) if is_async and user is not None: - callback.apply_async(( - user.username, xfxs.path, xfxs.photos, xfxs.osm, status - ), queue='instances') + callback.apply_async( + (user.username, xfxs.path, xfxs.photos, xfxs.osm, status), + queue="instances", + ) success_count += 1 else: try: success_count += callback(xfxs) except Exception as e: errors.append("%s => %s" % (xfxs.filename, str(e))) - del(xfxs) + del xfxs total_file_count += 1 return (total_file_count, success_count, errors) @@ -113,7 +116,7 @@ def import_instances_from_zip(zipfile_path, user, status="zip"): zf.extractall(temp_directory) except zipfile.BadZipfile as e: - errors = [u"%s" % e] + errors = ["%s" % e] return 0, 0, errors else: return import_instances_from_path(temp_directory, user, status) @@ -127,24 +130,15 @@ def callback(xform_fs): This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ - import_instance(user.username, - xform_fs.path, - xform_fs.photos, - xform_fs.osm, - status, - True) + import_instance( + user.username, xform_fs.path, xform_fs.photos, xform_fs.osm, status, True + ) if is_async: total_count, success_count, errors = iterate_through_instances( - path, - import_instance_async, - user=user, - status=status, - is_async=is_async + path, import_instance_async, user=user, status=status, is_async=is_async ) else: - total_count, success_count, errors = iterate_through_instances( - path, callback - ) + total_count, success_count, errors = iterate_through_instances(path, callback) return (total_count, success_count, errors) diff --git a/onadata/apps/logger/management/commands/add_id.py b/onadata/apps/logger/management/commands/add_id.py index 048a66ef16..fe8651a7d8 100644 --- a/onadata/apps/logger/management/commands/add_id.py +++ b/onadata/apps/logger/management/commands/add_id.py @@ -8,7 +8,7 @@ class Command(BaseCommand): - args = '' + args = "" help = gettext_lazy("Sync account with '_id'") def handle(self, *args, **kwargs): @@ -18,7 +18,7 @@ def handle(self, *args, **kwargs): users = User.objects.filter(username__contains=args[0]) else: # All the accounts - self.stdout.write("Fetching all the account {}", ending='\n') + self.stdout.write("Fetching all the account {}", ending="\n") users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ) @@ -27,24 +27,27 @@ def handle(self, *args, **kwargs): self.add_id(user) def add_id(self, user): - self.stdout.write("Syncing for account {}".format(user.username), - ending='\n') + self.stdout.write("Syncing for account {}".format(user.username), ending="\n") xforms = XForm.objects.filter(user=user) count = 0 failed = 0 - for instance in Instance.objects.filter( - xform__downloadable=True, xform__in=xforms)\ - .extra(where=['("logger_instance".json->>%s) is null'], - params=["_id"]).iterator(): + for instance in ( + Instance.objects.filter(xform__downloadable=True, xform__in=xforms) + .extra(where=['("logger_instance".json->>%s) is null'], params=["_id"]) + .iterator() + ): try: instance.save() count += 1 except Exception as e: failed += 1 - self.stdout.write(str(e), ending='\n') + self.stdout.write(str(e), ending="\n") pass - self.stdout.write("Syncing for account {}. Done. Success {}, Fail {}" - .format(user.username, count, failed), - ending='\n') + self.stdout.write( + "Syncing for account {}. Done. Success {}, Fail {}".format( + user.username, count, failed + ), + ending="\n", + ) diff --git a/onadata/apps/logger/management/commands/create_backup.py b/onadata/apps/logger/management/commands/create_backup.py index 29c91941a7..a6a54fde72 100644 --- a/onadata/apps/logger/management/commands/create_backup.py +++ b/onadata/apps/logger/management/commands/create_backup.py @@ -10,23 +10,22 @@ class Command(BaseCommand): args = "outfile username [id_string]" - help = gettext_lazy( - "Create a zip backup of a form and all its submissions") + help = gettext_lazy("Create a zip backup of a form and all its submissions") def handle(self, *args, **options): try: output_file = args[0] except IndexError: - raise CommandError(_("Provide the path to the zip file to backup" - " to")) + raise CommandError(_("Provide the path to the zip file to backup" " to")) else: output_file = os.path.realpath(output_file) try: username = args[1] except IndexError: - raise CommandError(_("You must provide the username to publish the" - " form to.")) + raise CommandError( + _("You must provide the username to publish the" " form to.") + ) # make sure user exists try: user = User.objects.get(username=username) @@ -42,6 +41,5 @@ def handle(self, *args, **options): try: xform = XForm.objects.get(user=user, id_string=id_string) except XForm.DoesNotExist: - raise CommandError(_("The id_string '%s' does not exist.") % - id_string) + raise CommandError(_("The id_string '%s' does not exist.") % id_string) create_zip_backup(output_file, user, xform) diff --git a/onadata/apps/logger/management/commands/export_gps_points.py b/onadata/apps/logger/management/commands/export_gps_points.py index 9e32605fa4..fec5259d0a 100644 --- a/onadata/apps/logger/management/commands/export_gps_points.py +++ b/onadata/apps/logger/management/commands/export_gps_points.py @@ -12,16 +12,16 @@ class Command(BaseCommand): help = gettext_lazy("Export all gps points with their timestamps") def handle(self, *args, **kwargs): - with open('gps_points_export.csv', 'w') as csvfile: - fieldnames = ['longitude', 'latitude', 'date_created'] + with open("gps_points_export.csv", "w") as csvfile: + fieldnames = ["longitude", "latitude", "date_created"] writer = csv.writer(csvfile) writer.writerow(fieldnames) for instance in queryset_iterator( - Instance.objects.exclude(geom__isnull=True)): - if hasattr(instance, 'point') and instance.point is not None: + Instance.objects.exclude(geom__isnull=True) + ): + if hasattr(instance, "point") and instance.point is not None: longitude = instance.point.coords[0] latitude = instance.point.coords[1] - writer.writerow( - [longitude, latitude, instance.date_created]) + writer.writerow([longitude, latitude, instance.date_created]) self.stdout.write("Export of gps files has completed!!!!") diff --git a/onadata/apps/logger/management/commands/fix_attachments_counts.py b/onadata/apps/logger/management/commands/fix_attachments_counts.py index 84c008f9b3..edc2b203a6 100644 --- a/onadata/apps/logger/management/commands/fix_attachments_counts.py +++ b/onadata/apps/logger/management/commands/fix_attachments_counts.py @@ -22,7 +22,8 @@ def update_attachments(instance): """ for attachment in instance.attachments.all(): attachment.name = os.path.basename( - get_original_filename(attachment.media_file.name)) + get_original_filename(attachment.media_file.name) + ) attachment.save() update_attachment_tracking(instance) @@ -31,18 +32,20 @@ class Command(BaseCommand): """ Fix attachments count command. """ - args = 'username' + + args = "username" help = gettext_lazy("Fix attachments count.") def add_arguments(self, parser): - parser.add_argument('username') + parser.add_argument("username") def handle(self, *args, **options): try: - username = options['username'] + username = options["username"] except KeyError: raise CommandError( - _("You must provide the username to publish the form to.")) + _("You must provide the username to publish the form to.") + ) # make sure user exists try: user = User.objects.get(username=username) @@ -56,16 +59,17 @@ def process_attachments(self, user): """ Process attachments for submissions where media_all_received is False. """ - xforms = XForm.objects.filter(user=user, deleted_at__isnull=True, - downloadable=True) + xforms = XForm.objects.filter( + user=user, deleted_at__isnull=True, downloadable=True + ) for xform in queryset_iterator(xforms): submissions = xform.instances.filter(media_all_received=False) to_process = submissions.count() if to_process: for submission in queryset_iterator(submissions): update_attachments(submission) - not_processed = xform.instances.filter( - media_all_received=False).count() - self.stdout.write("%s to process %s - %s = %s processed" % ( - xform, to_process, not_processed, - (to_process - not_processed))) + not_processed = xform.instances.filter(media_all_received=False).count() + self.stdout.write( + "%s to process %s - %s = %s processed" + % (xform, to_process, not_processed, (to_process - not_processed)) + ) diff --git a/onadata/apps/logger/management/commands/fix_duplicate_instances.py b/onadata/apps/logger/management/commands/fix_duplicate_instances.py index fe79ca0515..2dfab73a58 100644 --- a/onadata/apps/logger/management/commands/fix_duplicate_instances.py +++ b/onadata/apps/logger/management/commands/fix_duplicate_instances.py @@ -18,14 +18,17 @@ def query_data(self, sql): yield row def handle(self, *args, **kwargs): - sql = "select xform_id, uuid, COUNT(xform_id || uuid) "\ - "from logger_instance group by xform_id, uuid "\ - "HAVING COUNT(xform_id || uuid) > 1;" + sql = ( + "select xform_id, uuid, COUNT(xform_id || uuid) " + "from logger_instance group by xform_id, uuid " + "HAVING COUNT(xform_id || uuid) > 1;" + ) total_count = 0 total_deleted = 0 for xform, uuid, dupes_count in self.query_data(sql): - instances = Instance.objects.filter(xform_id=xform, uuid=uuid)\ - .order_by('pk') + instances = Instance.objects.filter(xform_id=xform, uuid=uuid).order_by( + "pk" + ) first = instances[0] xml = instances[0].xml is_mspray_form = xform == 80970 @@ -44,9 +47,7 @@ def handle(self, *args, **kwargs): else: to_delete = instances.exclude(pk=first.pk) - media_files = list( - first.attachments.values_list('media_file', flat=True) - ) + media_files = list(first.attachments.values_list("media_file", flat=True)) delete_count = 0 for i in to_delete: delete_count += 1 @@ -58,12 +59,13 @@ def handle(self, *args, **kwargs): if delete_count >= dupes_count: raise AssertionError( "# of records to delete %d should be less than total # of " - "duplicates %d." % (delete_count, dupes_count)) + "duplicates %d." % (delete_count, dupes_count) + ) to_delete.delete() total_count += dupes_count total_deleted += delete_count - self.stdout.write("deleted %d: %s (%d of %d)." - % (xform, uuid, delete_count, dupes_count)) + self.stdout.write( + "deleted %d: %s (%d of %d)." % (xform, uuid, delete_count, dupes_count) + ) - self.stdout.write("done: deleted %d of %d" - % (total_deleted, total_count)) + self.stdout.write("done: deleted %d of %d" % (total_deleted, total_count)) diff --git a/onadata/apps/logger/management/commands/fix_submission_count.py b/onadata/apps/logger/management/commands/fix_submission_count.py index 06e705d03d..5cc99a0c17 100644 --- a/onadata/apps/logger/management/commands/fix_submission_count.py +++ b/onadata/apps/logger/management/commands/fix_submission_count.py @@ -18,25 +18,28 @@ def handle(self, *args, **kwargs): xform_count = XForm.objects.filter(downloadable=True).count() for xform in XForm.objects.filter(downloadable=True).iterator(): with transaction.atomic(): - instance_count = xform.instances.filter(deleted_at=None)\ - .count() + instance_count = xform.instances.filter(deleted_at=None).count() xform.num_of_submissions = instance_count - xform.save(update_fields=['num_of_submissions']) + xform.save(update_fields=["num_of_submissions"]) i += 1 - self.stdout.write('Processing {} of {}: {} ({})'.format( - i, xform_count, xform.id_string, instance_count)) + self.stdout.write( + "Processing {} of {}: {} ({})".format( + i, xform_count, xform.id_string, instance_count + ) + ) i = 0 profile_count = UserProfile.objects.count() - for profile in UserProfile.objects.select_related('user__username')\ - .iterator(): + for profile in UserProfile.objects.select_related("user__username").iterator(): with transaction.atomic(): instance_count = Instance.objects.filter( - deleted_at=None, - xform__user_id=profile.user_id + deleted_at=None, xform__user_id=profile.user_id ).count() profile.num_of_submissions = instance_count - profile.save(update_fields=['num_of_submissions']) + profile.save(update_fields=["num_of_submissions"]) i += 1 - self.stdout.write('Processing {} of {}: {} ({})'.format( - i, profile_count, profile.user.username, instance_count)) + self.stdout.write( + "Processing {} of {}: {} ({})".format( + i, profile_count, profile.user.username, instance_count + ) + ) diff --git a/onadata/apps/logger/management/commands/generate_platform_stats.py b/onadata/apps/logger/management/commands/generate_platform_stats.py index 093377ad08..b550cc10de 100644 --- a/onadata/apps/logger/management/commands/generate_platform_stats.py +++ b/onadata/apps/logger/management/commands/generate_platform_stats.py @@ -16,8 +16,7 @@ def _write_stats_to_file(month: int, year: int): - out_file = open( - f"/tmp/platform_statistics_{month}_{year}.csv", "w") # nosec + out_file = open(f"/tmp/platform_statistics_{month}_{year}.csv", "w") # nosec writer = csv.writer(out_file) headers = ["Username", "Project Name", "Form Title", "No. of submissions"] writer.writerow(headers) @@ -26,17 +25,23 @@ def _write_stats_to_file(month: int, year: int): forms = XForm.objects.filter( Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - date_created__lte=date_obj - ).values('id', 'project__name', 'project__organization__username', 'title') + date_created__lte=date_obj, + ).values("id", "project__name", "project__organization__username", "title") with use_master: for form in forms: instance_count = Instance.objects.filter( Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - xform_id=form.get('id'), date_created__lte=date_obj + xform_id=form.get("id"), + date_created__lte=date_obj, ).count() writer.writerow( - [form.get('project__organization__username'), - form.get('project__name'), form.get('title'), instance_count]) + [ + form.get("project__organization__username"), + form.get("project__name"), + form.get("title"), + instance_count, + ] + ) class Command(BaseCommand): @@ -45,27 +50,30 @@ class Command(BaseCommand): information about the number of organizations, users, projects & submissions """ + help = _("Generates system statistics for the entire platform") def add_arguments(self, parser): parser.add_argument( - '--month', - '-m', - dest='month', - help=('Month to calculate system statistics for.' - 'Defaults to current month.'), - default=None + "--month", + "-m", + dest="month", + help=( + "Month to calculate system statistics for." "Defaults to current month." + ), + default=None, ) parser.add_argument( - '--year', - '-y', - dest='year', - help=('Year to calculate system statistics for.' - ' Defaults to current year'), + "--year", + "-y", + dest="year", + help=( + "Year to calculate system statistics for." " Defaults to current year" + ), default=None, ) def handle(self, *args, **options): - month = int(options.get('month', datetime.now().month)) - year = int(options.get('year', datetime.now().year)) + month = int(options.get("month", datetime.now().month)) + year = int(options.get("year", datetime.now().year)) _write_stats_to_file(month, year) diff --git a/onadata/apps/logger/management/commands/import.py b/onadata/apps/logger/management/commands/import.py index fe1894423c..f557c1788c 100644 --- a/onadata/apps/logger/management/commands/import.py +++ b/onadata/apps/logger/management/commands/import.py @@ -12,5 +12,5 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): path = args[0] - call_command('import_forms', os.path.join(path, "forms")) - call_command('import_instances', os.path.join(path, "instances")) + call_command("import_forms", os.path.join(path, "forms")) + call_command("import_instances", os.path.join(path, "instances")) diff --git a/onadata/apps/logger/management/commands/import_tools.py b/onadata/apps/logger/management/commands/import_tools.py index 399ae0b609..d01e3e1968 100644 --- a/onadata/apps/logger/management/commands/import_tools.py +++ b/onadata/apps/logger/management/commands/import_tools.py @@ -21,26 +21,29 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): if args.__len__() < 2: - raise CommandError(_(u"path(xform instances) username")) + raise CommandError(_("path(xform instances) username")) path = args[0] username = args[1] try: user = User.objects.get(username=username) except User.DoesNotExist: - raise CommandError(_(u"Invalid username %s") % username) + raise CommandError(_("Invalid username %s") % username) debug = False if debug: - self.stdout.write(_(u"[Importing XForm Instances from %(path)s]\n") - % {'path': path}) - im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) - self.stdout.write(_(u"Before Parse:")) - self.stdout.write(_(u" --> Images: %(nb)d") % {'nb': im_count}) - self.stdout.write((_(u" --> Instances: %(nb)d") - % {'nb': Instance.objects.count()})) + self.stdout.write( + _("[Importing XForm Instances from %(path)s]\n") % {"path": path} + ) + im_count = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) + self.stdout.write(_("Before Parse:")) + self.stdout.write(_(" --> Images: %(nb)d") % {"nb": im_count}) + self.stdout.write( + (_(" --> Instances: %(nb)d") % {"nb": Instance.objects.count()}) + ) import_instances_from_zip(path, user) if debug: - im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) - self.stdout.write(_(u"After Parse:")) - self.stdout.write(_(u" --> Images: %(nb)d") % {'nb': im_count2}) - self.stdout.write((_(u" --> Instances: %(nb)d") - % {'nb': Instance.objects.count()})) + im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) + self.stdout.write(_("After Parse:")) + self.stdout.write(_(" --> Images: %(nb)d") % {"nb": im_count2}) + self.stdout.write( + (_(" --> Instances: %(nb)d") % {"nb": Instance.objects.count()}) + ) diff --git a/onadata/apps/logger/management/commands/pull_from_aggregate.py b/onadata/apps/logger/management/commands/pull_from_aggregate.py index d1dfb1a2ad..88217bc7e6 100644 --- a/onadata/apps/logger/management/commands/pull_from_aggregate.py +++ b/onadata/apps/logger/management/commands/pull_from_aggregate.py @@ -11,23 +11,24 @@ class Command(BaseCommand): help = _("Insert all existing parsed instances into MongoDB") def add_arguments(self, parser): - parser.add_argument( - '--url', help=_("server url to pull forms and submissions")) - parser.add_argument('-u', '--username', help=_("Username")) - parser.add_argument('-p', '--password', help=_("Password")) - parser.add_argument('--to', help=_("username in this server")) + parser.add_argument("--url", help=_("server url to pull forms and submissions")) + parser.add_argument("-u", "--username", help=_("Username")) + parser.add_argument("-p", "--password", help=_("Password")) + parser.add_argument("--to", help=_("username in this server")) def handle(self, *args, **kwargs): - url = kwargs.get('url') - username = kwargs.get('username') - password = kwargs.get('password') - to = kwargs.get('to') + url = kwargs.get("url") + username = kwargs.get("username") + password = kwargs.get("password") + to = kwargs.get("to") if username is None or password is None or to is None or url is None: self.stderr.write( - 'pull_form_aggregate -u username -p password --to=username' - ' --url=aggregate_server_url') + "pull_form_aggregate -u username -p password --to=username" + " --url=aggregate_server_url" + ) else: user = User.objects.get(username=to) bc = BriefcaseClient( - username=username, password=password, user=user, url=url) + username=username, password=password, user=user, url=url + ) bc.download_xforms(include_instances=True) diff --git a/onadata/apps/logger/management/commands/recover_deleted_attachments.py b/onadata/apps/logger/management/commands/recover_deleted_attachments.py index fa1bc6629f..3199f68ec1 100644 --- a/onadata/apps/logger/management/commands/recover_deleted_attachments.py +++ b/onadata/apps/logger/management/commands/recover_deleted_attachments.py @@ -18,23 +18,22 @@ def recover_deleted_attachments(form_id: str, stdout=None): :param: (str) form_id: Unique identifier for an XForm object :param: (sys.stdout) stdout: Python standard output. Default: None """ - instances = Instance.objects.filter( - xform__id=form_id, deleted_at__isnull=True) + instances = Instance.objects.filter(xform__id=form_id, deleted_at__isnull=True) for instance in instances: expected_attachments = instance.get_expected_media() - if not instance.attachments.filter( - deleted_at__isnull=True).count() == len(expected_attachments): + if not instance.attachments.filter(deleted_at__isnull=True).count() == len( + expected_attachments + ): attachments_to_recover = instance.attachments.filter( - deleted_at__isnull=False, - name__in=expected_attachments) + deleted_at__isnull=False, name__in=expected_attachments + ) for attachment in attachments_to_recover: attachment.deleted_at = None attachment.deleted_by = None attachment.save() if stdout: - stdout.write( - f'Recovered {attachment.name} ID: {attachment.id}') + stdout.write(f"Recovered {attachment.name} ID: {attachment.id}") # Regenerate instance JSON instance.json = instance.get_full_dict(load_existing=False) instance.save() @@ -45,11 +44,12 @@ class Command(BaseCommand): Management command used to recover wrongfully deleted attachments. """ - help = 'Restore wrongly deleted attachments' + + help = "Restore wrongly deleted attachments" def add_arguments(self, parser): - parser.add_argument('-f', '--form', dest='form_id', type=int) + parser.add_argument("-f", "--form", dest="form_id", type=int) def handle(self, *args, **options): - form_id = options.get('form_id') + form_id = options.get("form_id") recover_deleted_attachments(form_id, self.stdout) diff --git a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py index ac99aba59a..a9a8fc3702 100644 --- a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py +++ b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py @@ -7,13 +7,12 @@ from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml -def _traverse_child_nodes_and_delete_column( - xml_obj, column: str) -> None: +def _traverse_child_nodes_and_delete_column(xml_obj, column: str) -> None: childNodes = xml_obj.childNodes for elem in childNodes: if elem.nodeName in column: xml_obj.removeChild(elem) - if hasattr(elem, 'childNodes'): + if hasattr(elem, "childNodes"): _traverse_child_nodes_and_delete_column(elem, column) @@ -25,48 +24,45 @@ def remove_columns_from_xml(xml: str, columns: List[str]) -> str: class Command(BaseCommand): - help = _( - 'Delete specific columns from submission ' - 'XMLs pulled by ODK Briefcase.') + help = _("Delete specific columns from submission " "XMLs pulled by ODK Briefcase.") def add_arguments(self, parser): parser.add_argument( - '--input', - '-i', - dest='in_dir', - help='Path to instances directory to pull submission XMLs from.' + "--input", + "-i", + dest="in_dir", + help="Path to instances directory to pull submission XMLs from.", ) parser.add_argument( - '--output', - '-o', - default='replaced-submissions', - dest='out_dir', - help='Path to directory to output modified submission XMLs' + "--output", + "-o", + default="replaced-submissions", + dest="out_dir", + help="Path to directory to output modified submission XMLs", ) parser.add_argument( - '--columns', - '-c', - dest='columns', - help='Comma separated list of columns to remove from the XMLs' + "--columns", + "-c", + dest="columns", + help="Comma separated list of columns to remove from the XMLs", ) parser.add_argument( - '--overwrite', - '-f', + "--overwrite", + "-f", default=False, - dest='overwrite', - action='store_true', - help='Whether to overwrite the original submission' + dest="overwrite", + action="store_true", + help="Whether to overwrite the original submission", ) def handle(self, *args, **options): - columns: List[str] = options.get('columns').split(',') - in_dir: str = options.get('in_dir') - out_dir: str = options.get('out_dir') - overwrite: bool = options.get('overwrite') + columns: List[str] = options.get("columns").split(",") + in_dir: str = options.get("in_dir") + out_dir: str = options.get("out_dir") + overwrite: bool = options.get("overwrite") submission_folders = [ - xml_file for xml_file in os.listdir(in_dir) - if xml_file.startswith('uuid') + xml_file for xml_file in os.listdir(in_dir) if xml_file.startswith("uuid") ] total_files = len(submission_folders) modified_files = 0 @@ -76,36 +72,33 @@ def handle(self, *args, **options): for count, submission_folder in enumerate(submission_folders, start=1): self.stdout.write( - f'Modifying {submission_folder}. ' - f'Progress {count}/{total_files}') + f"Modifying {submission_folder}. " f"Progress {count}/{total_files}" + ) data = None - with open( - f'{in_dir}/{submission_folder}/submission.xml', - 'r') as in_file: - data = in_file.read().replace('\n', '') + with open(f"{in_dir}/{submission_folder}/submission.xml", "r") as in_file: + data = in_file.read().replace("\n", "") data = remove_columns_from_xml(data, columns) in_file.close() remove_columns_from_xml(data, columns) if not overwrite: - os.makedirs(f'{out_dir}/{submission_folder}') + os.makedirs(f"{out_dir}/{submission_folder}") with open( - f'{out_dir}/{submission_folder}/submission.xml', - 'w') as out_file: + f"{out_dir}/{submission_folder}/submission.xml", "w" + ) as out_file: out_file.write(data) out_file.close() else: with open( - f'{in_dir}/{submission_folder}/submission.xml', - 'r+')as out_file: + f"{in_dir}/{submission_folder}/submission.xml", "r+" + ) as out_file: out_file.truncate(0) out_file.write(data) out_file.close() modified_files += 1 - self.stdout.write( - f'Operation completed. Modified {modified_files} files.') + self.stdout.write(f"Operation completed. Modified {modified_files} files.") diff --git a/onadata/apps/logger/management/commands/replace_form_id_root_node.py b/onadata/apps/logger/management/commands/replace_form_id_root_node.py index 9d928ac88a..d4f5f47562 100644 --- a/onadata/apps/logger/management/commands/replace_form_id_root_node.py +++ b/onadata/apps/logger/management/commands/replace_form_id_root_node.py @@ -16,7 +16,8 @@ def replace_form_id_with_correct_root_node( - inst_id: int, root: str = None, commit: bool = False) -> str: + inst_id: int, root: str = None, commit: bool = False +) -> str: inst: Instance = Instance.objects.get(id=inst_id, deleted_at__isnull=True) initial_xml = inst.xml form_id = re.escape(inst.xform.id_string) @@ -25,8 +26,8 @@ def replace_form_id_with_correct_root_node( opening_tag_regex = f"<{form_id}" closing_tag_regex = f"" - edited_xml = re.sub(opening_tag_regex, f'<{root}', initial_xml) - edited_xml = re.sub(closing_tag_regex, f'', edited_xml) + edited_xml = re.sub(opening_tag_regex, f"<{root}", initial_xml) + edited_xml = re.sub(closing_tag_regex, f"", edited_xml) if commit: last_edited = timezone.now() @@ -36,7 +37,7 @@ def replace_form_id_with_correct_root_node( xform_instance=inst, ) inst.last_edited = last_edited - inst.checksum = sha256(edited_xml.encode('utf-8')).hexdigest() + inst.checksum = sha256(edited_xml.encode("utf-8")).hexdigest() inst.xml = edited_xml inst.save() return f"Modified Instance ID {inst.id} - History object {history.id}" @@ -49,39 +50,40 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--instance-ids', - '-i', - dest='instance_ids', - help='Comma-separated list of instance ids.' + "--instance-ids", + "-i", + dest="instance_ids", + help="Comma-separated list of instance ids.", ) parser.add_argument( - '--commit-changes', - '-c', - action='store_true', - dest='commit', + "--commit-changes", + "-c", + action="store_true", + dest="commit", default=False, - help='Save XML changes' + help="Save XML changes", ) parser.add_argument( - '--root-node', - '-r', - dest='root', + "--root-node", + "-r", + dest="root", default=None, - help='Default root node name to replace the form ID with' + help="Default root node name to replace the form ID with", ) def handle(self, *args, **options): - instance_ids = options.get('instance_ids').split(',') - commit = options.get('commit') - root = options.get('root') + instance_ids = options.get("instance_ids").split(",") + commit = options.get("commit") + root = options.get("root") if not instance_ids: - raise CommandError('No instance id provided.') + raise CommandError("No instance id provided.") for inst_id in instance_ids: try: msg = replace_form_id_with_correct_root_node( - inst_id, root=root, commit=commit) + inst_id, root=root, commit=commit + ) except Instance.DoesNotExist: msg = f"Instance with ID {inst_id} does not exist" diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py index f6e3445006..df351dddae 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py @@ -13,10 +13,13 @@ class Command(BaseCommand): help = gettext_lazy("Set xform.instances_with_osm") def handle(self, *args, **kwargs): - pks = Attachment.objects.filter( - extension=Attachment.OSM, - instance__xform__instances_with_osm=False)\ - .values_list('instance__xform', flat=True).distinct() + pks = ( + Attachment.objects.filter( + extension=Attachment.OSM, instance__xform__instances_with_osm=False + ) + .values_list("instance__xform", flat=True) + .distinct() + ) xforms = XForm.objects.filter(pk__in=pks) total = xforms.count() count = 0 diff --git a/onadata/apps/logger/management/commands/transferproject.py b/onadata/apps/logger/management/commands/transferproject.py index 56d86e9a2d..1f8548237a 100644 --- a/onadata/apps/logger/management/commands/transferproject.py +++ b/onadata/apps/logger/management/commands/transferproject.py @@ -4,8 +4,9 @@ from django.db import transaction from onadata.apps.logger.models import Project, XForm, DataView, MergedXForm -from onadata.apps.logger.models.project import set_object_permissions \ - as set_project_permissions +from onadata.apps.logger.models.project import ( + set_object_permissions as set_project_permissions, +) from onadata.libs.utils.project_utils import set_project_perms_to_xform @@ -19,35 +20,36 @@ class Command(BaseCommand): Depending on what is supplied for --project-id or --all-projects, the command will either transfer a single project or all the projects. """ - help = 'A command to reassign a project(s) from one user to the other.' + + help = "A command to reassign a project(s) from one user to the other." errors = [] def add_arguments(self, parser): parser.add_argument( - '--current-owner', - dest='current_owner', + "--current-owner", + dest="current_owner", type=str, - help='Username of the current owner of the project(s)', + help="Username of the current owner of the project(s)", ) parser.add_argument( - '--new-owner', - dest='new_owner', + "--new-owner", + dest="new_owner", type=str, - help='Username of the new owner of the project(s)', + help="Username of the new owner of the project(s)", ) parser.add_argument( - '--project-id', - dest='project_id', + "--project-id", + dest="project_id", type=int, - help='Id of the project to be transferred.', + help="Id of the project to be transferred.", ) parser.add_argument( - '--all-projects', - dest='all_projects', - action='store_true', - help='Supply this command if all the projects are to be' - ' transferred. If not, do not include the argument', + "--all-projects", + dest="all_projects", + action="store_true", + help="Supply this command if all the projects are to be" + " transferred. If not, do not include the argument", ) def get_user(self, username): # pylint: disable=C0111 @@ -65,7 +67,8 @@ def update_xform_with_new_user(self, project, user): for the xForm and the project. """ xforms = XForm.objects.filter( - project=project, deleted_at__isnull=True, downloadable=True) + project=project, deleted_at__isnull=True, downloadable=True + ) for form in xforms: form.user = user form.created_by = user @@ -75,9 +78,10 @@ def update_xform_with_new_user(self, project, user): @staticmethod def update_data_views(form): - """Update DataView project for the XForm given. """ + """Update DataView project for the XForm given.""" dataviews = DataView.objects.filter( - xform=form, project=form.project, deleted_at__isnull=True) + xform=form, project=form.project, deleted_at__isnull=True + ) for data_view in dataviews: data_view.project = form.project data_view.save() @@ -86,7 +90,8 @@ def update_data_views(form): def update_merged_xform(project, user): """Update ownership of MergedXforms.""" merged_xforms = MergedXForm.objects.filter( - project=project, deleted_at__isnull=True) + project=project, deleted_at__isnull=True + ) for form in merged_xforms: form.user = user form.created_by = user @@ -96,13 +101,13 @@ def update_merged_xform(project, user): @transaction.atomic() def handle(self, *args, **options): """Transfer projects from one user to another.""" - from_user = self.get_user(options['current_owner']) - to_user = self.get_user(options['new_owner']) - project_id = options.get('project_id') - transfer_all_projects = options.get('all_projects') + from_user = self.get_user(options["current_owner"]) + to_user = self.get_user(options["new_owner"]) + project_id = options.get("project_id") + transfer_all_projects = options.get("all_projects") if self.errors: - self.stdout.write(''.join(self.errors)) + self.stdout.write("".join(self.errors)) return # No need to validate project ownership as they filtered @@ -110,10 +115,10 @@ def handle(self, *args, **options): projects = [] if transfer_all_projects: projects = Project.objects.filter( - organization=from_user, deleted_at__isnull=True) + organization=from_user, deleted_at__isnull=True + ) else: - projects = Project.objects.filter( - id=project_id, organization=from_user) + projects = Project.objects.filter(id=project_id, organization=from_user) for project in projects: project.organization = to_user @@ -124,4 +129,4 @@ def handle(self, *args, **options): self.update_merged_xform(project, to_user) set_project_permissions(Project, project, created=True) - self.stdout.write('Projects transferred successfully') + self.stdout.write("Projects transferred successfully") diff --git a/onadata/apps/logger/management/commands/update_xform_uuids.py b/onadata/apps/logger/management/commands/update_xform_uuids.py index ff08e0428b..42840a487b 100644 --- a/onadata/apps/logger/management/commands/update_xform_uuids.py +++ b/onadata/apps/logger/management/commands/update_xform_uuids.py @@ -6,26 +6,28 @@ from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext_lazy -from onadata.apps.logger.models.xform import (DuplicateUUIDError, XForm, - update_xform_uuid) +from onadata.apps.logger.models.xform import ( + DuplicateUUIDError, + XForm, + update_xform_uuid, +) class Command(BaseCommand): help = gettext_lazy( - "Use a csv file with username, id_string and new_uuid to set new" - " uuids") + "Use a csv file with username, id_string and new_uuid to set new" " uuids" + ) def add_arguments(self, parser): - parser.add_argument( - '-f', '--file', help=gettext_lazy("Path to csv file")) + parser.add_argument("-f", "--file", help=gettext_lazy("Path to csv file")) def handle(self, *args, **kwargs): # all options are required - if not kwargs.get('file'): + if not kwargs.get("file"): raise CommandError("You must provide a path to the csv file") # try open the file try: - with open(kwargs.get('file'), "r") as f: + with open(kwargs.get("file"), "r") as f: lines = csv.reader(f) i = 0 for line in lines: @@ -35,18 +37,18 @@ def handle(self, *args, **kwargs): uuid = line[2] update_xform_uuid(username, id_string, uuid) except IndexError: - self.stderr.write( - "line %d is in an invalid format" % (i + 1)) + self.stderr.write("line %d is in an invalid format" % (i + 1)) except XForm.DoesNotExist: - self.stderr.write("XForm with username: %s and id " - "string: %s does not exist" - % (username, id_string)) + self.stderr.write( + "XForm with username: %s and id " + "string: %s does not exist" % (username, id_string) + ) except DuplicateUUIDError: self.stderr.write( - "An xform with uuid: %s already exists" % uuid) + "An xform with uuid: %s already exists" % uuid + ) else: i += 1 self.stdout.write("Updated %d rows" % i) except IOError: - raise CommandError( - "file %s could not be open" % kwargs.get('file')) + raise CommandError("file %s could not be open" % kwargs.get("file")) diff --git a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py index 3dee4c6a7a..74bf18b414 100644 --- a/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py +++ b/onadata/apps/logger/migrations/0001_pre-django-3-upgrade.py @@ -21,16 +21,17 @@ def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 """ Recalculate all XForm hashes. """ - XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 - xforms = XForm.objects.filter(downloadable=True, - deleted_at__isnull=True).only('xml') + XForm = apps.get_model("logger", "XForm") # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, deleted_at__isnull=True).only( + "xml" + ) count = xforms.count() counter = 0 for xform in queryset_iterator(xforms, 500): - hash_value = md5(xform.xml.encode('utf8')).hexdigest() + hash_value = md5(xform.xml.encode("utf8")).hexdigest() xform.hash = f"md5:{hash_value}" - xform.save(update_fields=['hash']) + xform.save(update_fields=["hash"]) counter += 1 if counter % 500 == 0: print(f"Processed {counter} of {count} forms.") @@ -42,9 +43,9 @@ def generate_uuid_if_missing(apps, schema_editor): """ Generate uuids for XForms without them """ - XForm = apps.get_model('logger', 'XForm') + XForm = apps.get_model("logger", "XForm") - for xform in XForm.objects.filter(uuid=''): + for xform in XForm.objects.filter(uuid=""): xform.uuid = onadata.libs.utils.common_tools.get_uuid() xform.save() @@ -54,9 +55,10 @@ def regenerate_instance_json(apps, schema_editor): Regenerate Instance JSON """ for inst in Instance.objects.filter( - deleted_at__isnull=True, - xform__downloadable=True, - xform__deleted_at__isnull=True): + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True, + ): inst.json = inst.get_full_dict(load_existing=False) inst.save() @@ -67,8 +69,7 @@ def create_initial_xform_version(apps, schema_editor): Version """ queryset = onadata.apps.logger.models.xform.XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True + downloadable=True, deleted_at__isnull=True ) for xform in queryset.iterator(): if xform.version: @@ -76,462 +77,1191 @@ def create_initial_xform_version(apps, schema_editor): class Migration(migrations.Migration): - replaces = [('logger', '0001_initial'), ('logger', '0002_auto_20150717_0048'), ('logger', '0003_dataview_instances_with_geopoints'), ('logger', '0004_auto_20150910_0056'), ('logger', '0005_auto_20151015_0758'), ('logger', '0006_auto_20151106_0130'), ('logger', '0007_osmdata_field_name'), ('logger', '0008_osmdata_osm_type'), ('logger', '0009_auto_20151111_0438'), ('logger', '0010_attachment_file_size'), ('logger', '0011_dataview_matches_parent'), ('logger', '0012_auto_20160114_0708'), ('logger', '0013_note_created_by'), ('logger', '0014_note_instance_field'), ('logger', '0015_auto_20160222_0559'), ('logger', '0016_widget_aggregation'), ('logger', '0017_auto_20160224_0130'), ('logger', '0018_auto_20160301_0330'), ('logger', '0019_auto_20160307_0256'), ('logger', '0020_auto_20160408_0325'), ('logger', '0021_auto_20160408_0919'), ('logger', '0022_auto_20160418_0518'), ('logger', '0023_auto_20160419_0403'), ('logger', '0024_xform_has_hxl_support'), ('logger', '0025_xform_last_updated_at'), ('logger', '0026_auto_20160913_0239'), ('logger', '0027_auto_20161201_0730'), ('logger', '0028_auto_20170221_0838'), ('logger', '0029_auto_20170221_0908'), ('logger', '0030_auto_20170227_0137'), ('logger', '0028_auto_20170217_0502'), ('logger', '0031_merge'), ('logger', '0032_project_deleted_at'), ('logger', '0033_auto_20170705_0159'), ('logger', '0034_mergedxform'), ('logger', '0035_auto_20170712_0529'), ('logger', '0036_xform_is_merged_dataset'), ('logger', '0034_auto_20170814_0432'), ('logger', '0037_merge_20170825_0238'), ('logger', '0038_auto_20170828_1718'), ('logger', '0039_auto_20170909_2052'), ('logger', '0040_auto_20170912_1504'), ('logger', '0041_auto_20170912_1512'), ('logger', '0042_xform_hash'), ('logger', '0043_auto_20171010_0403'), ('logger', '0044_xform_hash_sql_update'), ('logger', '0045_attachment_name'), ('logger', '0046_auto_20180314_1618'), ('logger', '0047_dataview_deleted_at'), ('logger', '0048_dataview_deleted_by'), ('logger', '0049_xform_deleted_by'), ('logger', '0050_project_deleted_by'), ('logger', '0051_auto_20180522_1118'), ('logger', '0052_auto_20180805_2233'), ('logger', '0053_submissionreview'), ('logger', '0054_instance_has_a_review'), ('logger', '0055_auto_20180904_0713'), ('logger', '0056_auto_20190125_0517'), ('logger', '0057_xform_public_key'), ('logger', '0058_auto_20191211_0900'), ('logger', '0059_attachment_deleted_by'), ('logger', '0060_auto_20200305_0357'), ('logger', '0061_auto_20200713_0814'), ('logger', '0062_auto_20210202_0248'), ('logger', '0063_xformversion'), ('logger', '0064_auto_20210304_0314')] + replaces = [ + ("logger", "0001_initial"), + ("logger", "0002_auto_20150717_0048"), + ("logger", "0003_dataview_instances_with_geopoints"), + ("logger", "0004_auto_20150910_0056"), + ("logger", "0005_auto_20151015_0758"), + ("logger", "0006_auto_20151106_0130"), + ("logger", "0007_osmdata_field_name"), + ("logger", "0008_osmdata_osm_type"), + ("logger", "0009_auto_20151111_0438"), + ("logger", "0010_attachment_file_size"), + ("logger", "0011_dataview_matches_parent"), + ("logger", "0012_auto_20160114_0708"), + ("logger", "0013_note_created_by"), + ("logger", "0014_note_instance_field"), + ("logger", "0015_auto_20160222_0559"), + ("logger", "0016_widget_aggregation"), + ("logger", "0017_auto_20160224_0130"), + ("logger", "0018_auto_20160301_0330"), + ("logger", "0019_auto_20160307_0256"), + ("logger", "0020_auto_20160408_0325"), + ("logger", "0021_auto_20160408_0919"), + ("logger", "0022_auto_20160418_0518"), + ("logger", "0023_auto_20160419_0403"), + ("logger", "0024_xform_has_hxl_support"), + ("logger", "0025_xform_last_updated_at"), + ("logger", "0026_auto_20160913_0239"), + ("logger", "0027_auto_20161201_0730"), + ("logger", "0028_auto_20170221_0838"), + ("logger", "0029_auto_20170221_0908"), + ("logger", "0030_auto_20170227_0137"), + ("logger", "0028_auto_20170217_0502"), + ("logger", "0031_merge"), + ("logger", "0032_project_deleted_at"), + ("logger", "0033_auto_20170705_0159"), + ("logger", "0034_mergedxform"), + ("logger", "0035_auto_20170712_0529"), + ("logger", "0036_xform_is_merged_dataset"), + ("logger", "0034_auto_20170814_0432"), + ("logger", "0037_merge_20170825_0238"), + ("logger", "0038_auto_20170828_1718"), + ("logger", "0039_auto_20170909_2052"), + ("logger", "0040_auto_20170912_1504"), + ("logger", "0041_auto_20170912_1512"), + ("logger", "0042_xform_hash"), + ("logger", "0043_auto_20171010_0403"), + ("logger", "0044_xform_hash_sql_update"), + ("logger", "0045_attachment_name"), + ("logger", "0046_auto_20180314_1618"), + ("logger", "0047_dataview_deleted_at"), + ("logger", "0048_dataview_deleted_by"), + ("logger", "0049_xform_deleted_by"), + ("logger", "0050_project_deleted_by"), + ("logger", "0051_auto_20180522_1118"), + ("logger", "0052_auto_20180805_2233"), + ("logger", "0053_submissionreview"), + ("logger", "0054_instance_has_a_review"), + ("logger", "0055_auto_20180904_0713"), + ("logger", "0056_auto_20190125_0517"), + ("logger", "0057_xform_public_key"), + ("logger", "0058_auto_20191211_0900"), + ("logger", "0059_attachment_deleted_by"), + ("logger", "0060_auto_20200305_0357"), + ("logger", "0061_auto_20200713_0814"), + ("logger", "0062_auto_20210202_0248"), + ("logger", "0063_xformversion"), + ("logger", "0064_auto_20210304_0314"), + ] initial = True dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), + ("contenttypes", "0002_remove_content_type_name"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('auth', '0001_initial'), - ('taggit', '0001_initial'), - ('contenttypes', '0001_initial'), + ("auth", "0001_initial"), + ("taggit", "0001_initial"), + ("contenttypes", "0001_initial"), ] operations = [ migrations.CreateModel( - name='Project', + name="Project", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), - ('shared', models.BooleanField(default=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_owner', to=settings.AUTH_USER_MODEL)), - ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_org', to=settings.AUTH_USER_MODEL)), - ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), - ('user_stars', models.ManyToManyField(related_name='project_stars', to=settings.AUTH_USER_MODEL)), - ('deleted_at', models.DateTimeField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "metadata", + django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ("shared", models.BooleanField(default=False)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="project_owner", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="project_org", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user_stars", + models.ManyToManyField( + related_name="project_stars", to=settings.AUTH_USER_MODEL + ), + ), + ("deleted_at", models.DateTimeField(blank=True, null=True)), ], options={ - 'permissions': (('view_project', 'Can view project'), ('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data')), - 'unique_together': {('name', 'organization')}, + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ), + "unique_together": {("name", "organization")}, }, ), migrations.CreateModel( - name='SurveyType', + name="SurveyType", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('slug', models.CharField(max_length=100, unique=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("slug", models.CharField(max_length=100, unique=True)), ], ), migrations.CreateModel( - name='XForm', + name="XForm", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xls', models.FileField(null=True, upload_to=onadata.apps.logger.models.xform.upload_to)), - ('json', models.TextField(default='')), - ('description', models.TextField(blank=True, default='', null=True)), - ('xml', models.TextField()), - ('require_auth', models.BooleanField(default=False)), - ('shared', models.BooleanField(default=False)), - ('shared_data', models.BooleanField(default=False)), - ('downloadable', models.BooleanField(default=True)), - ('allows_sms', models.BooleanField(default=False)), - ('encrypted', models.BooleanField(default=False)), - ('sms_id_string', models.SlugField(default=b'', editable=False, max_length=100, verbose_name='SMS ID')), - ('id_string', models.SlugField(editable=False, max_length=100, verbose_name='ID')), - ('title', models.CharField(editable=False, max_length=255)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(blank=True, null=True)), - ('last_submission_time', models.DateTimeField(blank=True, null=True)), - ('has_start_time', models.BooleanField(default=False)), - ('uuid', models.CharField(default='', max_length=32)), - ('bamboo_dataset', models.CharField(default='', max_length=60)), - ('instances_with_geopoints', models.BooleanField(default=False)), - ('instances_with_osm', models.BooleanField(default=False)), - ('num_of_submissions', models.IntegerField(default=0)), - ('version', models.CharField(blank=True, max_length=255, null=True)), - ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), - ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), - ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='xforms', to=settings.AUTH_USER_MODEL)), - ('has_hxl_support', models.BooleanField(default=False)), - ('last_updated_at', models.DateTimeField(auto_now=True, default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc))), - ('is_merged_dataset', models.BooleanField(default=False)), - ('hash', models.CharField(blank=True, default=None, max_length=36, null=True, verbose_name='Hash')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "xls", + models.FileField( + null=True, upload_to=onadata.apps.logger.models.xform.upload_to + ), + ), + ("json", models.TextField(default="")), + ("description", models.TextField(blank=True, default="", null=True)), + ("xml", models.TextField()), + ("require_auth", models.BooleanField(default=False)), + ("shared", models.BooleanField(default=False)), + ("shared_data", models.BooleanField(default=False)), + ("downloadable", models.BooleanField(default=True)), + ("allows_sms", models.BooleanField(default=False)), + ("encrypted", models.BooleanField(default=False)), + ( + "sms_id_string", + models.SlugField( + default=b"", + editable=False, + max_length=100, + verbose_name="SMS ID", + ), + ), + ( + "id_string", + models.SlugField(editable=False, max_length=100, verbose_name="ID"), + ), + ("title", models.CharField(editable=False, max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(blank=True, null=True)), + ("last_submission_time", models.DateTimeField(blank=True, null=True)), + ("has_start_time", models.BooleanField(default=False)), + ("uuid", models.CharField(default="", max_length=32)), + ("bamboo_dataset", models.CharField(default="", max_length=60)), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("instances_with_osm", models.BooleanField(default=False)), + ("num_of_submissions", models.IntegerField(default=0)), + ("version", models.CharField(blank=True, max_length=255, null=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="xforms", + to=settings.AUTH_USER_MODEL, + ), + ), + ("has_hxl_support", models.BooleanField(default=False)), + ( + "last_updated_at", + models.DateTimeField( + auto_now=True, + default=datetime.datetime( + 2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc + ), + ), + ), + ("is_merged_dataset", models.BooleanField(default=False)), + ( + "hash", + models.CharField( + blank=True, + default=None, + max_length=36, + null=True, + verbose_name="Hash", + ), + ), ], options={ - 'ordering': ('pk',), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms', - 'permissions': (('view_xform', 'Can view associated data'), ('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), - 'unique_together': {('user', 'id_string', 'project'), ('user', 'sms_id_string', 'project')}, + "ordering": ("pk",), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "unique_together": { + ("user", "id_string", "project"), + ("user", "sms_id_string", "project"), + }, }, ), migrations.CreateModel( - name='Instance', + name="Instance", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('json', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), - ('xml', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('status', models.CharField(default='submitted_via_web', max_length=20)), - ('uuid', models.CharField(db_index=True, default='', max_length=249)), - ('version', models.CharField(max_length=255, null=True)), - ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(null=True, srid=4326)), - ('survey_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.surveytype')), - ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), - ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instances', to=settings.AUTH_USER_MODEL)), - ('xform', models.ForeignKey(default=328, on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform')), - ('last_edited', models.DateTimeField(default=None, null=True)), - ('media_all_received', models.NullBooleanField(default=True, verbose_name='Received All Media Attachemts')), - ('media_count', models.PositiveIntegerField(default=0, null=True, verbose_name='Received Media Attachments')), - ('total_media', models.PositiveIntegerField(default=0, null=True, verbose_name='Total Media Attachments')), - ('checksum', models.CharField(blank=True, db_index=True, max_length=64, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("json", django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ("xml", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "status", + models.CharField(default="submitted_via_web", max_length=20), + ), + ("uuid", models.CharField(db_index=True, default="", max_length=249)), + ("version", models.CharField(max_length=255, null=True)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + null=True, srid=4326 + ), + ), + ( + "survey_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="logger.surveytype", + ), + ), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + default=328, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.xform", + ), + ), + ("last_edited", models.DateTimeField(default=None, null=True)), + ( + "media_all_received", + models.NullBooleanField( + default=True, verbose_name="Received All Media Attachemts" + ), + ), + ( + "media_count", + models.PositiveIntegerField( + default=0, null=True, verbose_name="Received Media Attachments" + ), + ), + ( + "total_media", + models.PositiveIntegerField( + default=0, null=True, verbose_name="Total Media Attachments" + ), + ), + ( + "checksum", + models.CharField( + blank=True, db_index=True, max_length=64, null=True + ), + ), ], options={ - 'unique_together': {('xform', 'uuid')}, + "unique_together": {("xform", "uuid")}, }, ), migrations.CreateModel( - name='ProjectUserObjectPermission', + name="ProjectUserObjectPermission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), - ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('user', 'permission', 'content_object')}, + "abstract": False, + "unique_together": {("user", "permission", "content_object")}, }, ), migrations.CreateModel( - name='ProjectGroupObjectPermission', + name="ProjectGroupObjectPermission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group')), - ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="auth.group" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('group', 'permission', 'content_object')}, + "abstract": False, + "unique_together": {("group", "permission", "content_object")}, }, ), migrations.CreateModel( - name='XFormUserObjectPermission', + name="XFormUserObjectPermission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), - ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('user', 'permission', 'content_object')}, + "abstract": False, + "unique_together": {("user", "permission", "content_object")}, }, ), migrations.CreateModel( - name='XFormGroupObjectPermission', + name="XFormGroupObjectPermission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group')), - ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "content_object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="auth.group" + ), + ), + ( + "permission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="auth.permission", + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('group', 'permission', 'content_object')}, + "abstract": False, + "unique_together": {("group", "permission", "content_object")}, }, ), migrations.CreateModel( - name='Note', + name="Note", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('note', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notes', to='logger.instance')), - ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ('instance_field', models.TextField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("note", models.TextField()), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="notes", + to="logger.instance", + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ("instance_field", models.TextField(blank=True, null=True)), ], options={ - 'permissions': (('view_note', 'View note'),), + "permissions": (("view_note", "View note"),), }, ), migrations.CreateModel( - name='DataView', + name="DataView", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('columns', django.contrib.postgres.fields.jsonb.JSONField()), - ('query', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.project')), - ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='logger.xform')), - ('instances_with_geopoints', models.BooleanField(default=False)), - ('matches_parent', models.BooleanField(default=False)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("columns", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "query", + django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.project" + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="logger.xform" + ), + ), + ("instances_with_geopoints", models.BooleanField(default=False)), + ("matches_parent", models.BooleanField(default=False)), ], options={ - 'verbose_name': 'Data View', - 'verbose_name_plural': 'Data Views', + "verbose_name": "Data View", + "verbose_name_plural": "Data Views", }, ), migrations.CreateModel( - name='OsmData', + name="OsmData", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xml', models.TextField()), - ('osm_id', models.CharField(max_length=20)), - ('tags', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), - ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(srid=4326)), - ('filename', models.CharField(max_length=255)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='osm_data', to='logger.instance')), - ('field_name', models.CharField(blank=True, default=b'', max_length=255)), - ('osm_type', models.CharField(default=b'way', max_length=10)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xml", models.TextField()), + ("osm_id", models.CharField(max_length=20)), + ("tags", django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + srid=4326 + ), + ), + ("filename", models.CharField(max_length=255)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="osm_data", + to="logger.instance", + ), + ), + ( + "field_name", + models.CharField(blank=True, default=b"", max_length=255), + ), + ("osm_type", models.CharField(default=b"way", max_length=10)), ], options={ - 'unique_together': {('instance', 'field_name')}, + "unique_together": {("instance", "field_name")}, }, ), migrations.CreateModel( - name='Widget', + name="Widget", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('object_id', models.PositiveIntegerField()), - ('widget_type', models.CharField(choices=[(b'charts', b'Charts')], default=b'charts', max_length=25)), - ('view_type', models.CharField(max_length=50)), - ('column', models.CharField(max_length=255)), - ('group_by', models.CharField(blank=True, default=None, max_length=255, null=True)), - ('title', models.CharField(blank=True, default=None, max_length=255, null=True)), - ('description', models.CharField(blank=True, default=None, max_length=255, null=True)), - ('key', models.CharField(db_index=True, max_length=32, unique=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), - ('order', models.PositiveIntegerField(db_index=True, default=0, editable=False)), - ('aggregation', models.CharField(blank=True, default=None, max_length=255, null=True)), - ('metadata', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("object_id", models.PositiveIntegerField()), + ( + "widget_type", + models.CharField( + choices=[(b"charts", b"Charts")], + default=b"charts", + max_length=25, + ), + ), + ("view_type", models.CharField(max_length=50)), + ("column", models.CharField(max_length=255)), + ( + "group_by", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "title", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "description", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ("key", models.CharField(db_index=True, max_length=32, unique=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), + ( + "order", + models.PositiveIntegerField( + db_index=True, default=0, editable=False + ), + ), + ( + "aggregation", + models.CharField( + blank=True, default=None, max_length=255, null=True + ), + ), + ( + "metadata", + django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), + ), ], options={ - 'ordering': ('order',), + "ordering": ("order",), }, ), migrations.CreateModel( - name='OpenData', + name="OpenData", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('uuid', models.CharField(default=onadata.libs.utils.common_tools.get_uuid, max_length=32, unique=True)), - ('object_id', models.PositiveIntegerField(blank=True, null=True)), - ('active', models.BooleanField(default=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "uuid", + models.CharField( + default=onadata.libs.utils.common_tools.get_uuid, + max_length=32, + unique=True, + ), + ), + ("object_id", models.PositiveIntegerField(blank=True, null=True)), + ("active", models.BooleanField(default=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), ], ), migrations.CreateModel( - name='Attachment', + name="Attachment", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('media_file', models.FileField(max_length=255, upload_to=onadata.apps.logger.models.attachment.upload_to)), - ('mimetype', models.CharField(blank=True, default=b'', max_length=100)), - ('extension', models.CharField(db_index=True, default='non', max_length=10)), - ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='logger.instance')), - ('date_created', models.DateTimeField(auto_now_add=True, null=True)), - ('date_modified', models.DateTimeField(auto_now=True, null=True)), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('file_size', models.PositiveIntegerField(default=0)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "media_file", + models.FileField( + max_length=255, + upload_to=onadata.apps.logger.models.attachment.upload_to, + ), + ), + ("mimetype", models.CharField(blank=True, default=b"", max_length=100)), + ( + "extension", + models.CharField(db_index=True, default="non", max_length=10), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="attachments", + to="logger.instance", + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True, null=True)), + ("date_modified", models.DateTimeField(auto_now=True, null=True)), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ("file_size", models.PositiveIntegerField(default=0)), ], options={ - 'ordering': ('pk',), + "ordering": ("pk",), }, ), migrations.CreateModel( - name='MergedXForm', + name="MergedXForm", fields=[ - ('xform_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='logger.xform')), - ('xforms', models.ManyToManyField(related_name='mergedxform_ptr', to='logger.XForm')), + ( + "xform_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="logger.xform", + ), + ), + ( + "xforms", + models.ManyToManyField( + related_name="mergedxform_ptr", to="logger.XForm" + ), + ), ], options={ - 'permissions': (('view_mergedxform', 'Can view associated data'),), + "permissions": (("view_mergedxform", "Can view associated data"),), }, - bases=('logger.xform',), + bases=("logger.xform",), ), migrations.CreateModel( - name='InstanceHistory', + name="InstanceHistory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xml', models.TextField()), - ('uuid', models.CharField(default='', max_length=249)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xform_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='submission_history', to='logger.instance')), - ('geom', django.contrib.gis.db.models.fields.GeometryCollectionField(null=True, srid=4326)), - ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ('submission_date', models.DateTimeField(default=None, null=True)), - ('checksum', models.CharField(blank=True, max_length=64, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xml", models.TextField()), + ("uuid", models.CharField(default="", max_length=249)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "xform_instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="submission_history", + to="logger.instance", + ), + ), + ( + "geom", + django.contrib.gis.db.models.fields.GeometryCollectionField( + null=True, srid=4326 + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ("submission_date", models.DateTimeField(default=None, null=True)), + ("checksum", models.CharField(blank=True, max_length=64, null=True)), ], ), migrations.RunSQL( sql="UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", - reverse_sql='', + reverse_sql="", ), migrations.AddField( - model_name='attachment', - name='name', + model_name="attachment", + name="name", field=models.CharField(blank=True, max_length=100, null=True), ), migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(default="", max_length=36), ), migrations.AddField( - model_name='dataview', - name='deleted_at', + model_name="dataview", + name="deleted_at", field=models.DateTimeField(blank=True, null=True), ), migrations.AddField( - model_name='dataview', - name='deleted_by', - field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataview_deleted_by', to=settings.AUTH_USER_MODEL), + model_name="dataview", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="dataview_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AddField( - model_name='xform', - name='deleted_by', - field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), + model_name="xform", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="xform_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AddField( - model_name='project', - name='deleted_by', - field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_deleted_by', to=settings.AUTH_USER_MODEL), - ), - migrations.RunPython( - recalculate_xform_hash + model_name="project", + name="deleted_by", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="project_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), + migrations.RunPython(recalculate_xform_hash), migrations.AddField( - model_name='instance', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AddField( - model_name='instance', - name='has_a_review', - field=models.BooleanField(default=False, verbose_name='has_a_review'), + model_name="instance", + name="has_a_review", + field=models.BooleanField(default=False, verbose_name="has_a_review"), ), migrations.CreateModel( - name='SubmissionReview', + name="SubmissionReview", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.CharField(choices=[('1', 'Approved'), ('3', 'Pending'), ('2', 'Rejected')], db_index=True, default='3', max_length=1, verbose_name='Status')), - ('deleted_at', models.DateTimeField(db_index=True, default=None, null=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ('deleted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_reviews', to=settings.AUTH_USER_MODEL)), - ('instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='reviews', to='logger.instance')), - ('note', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='notes', to='logger.note')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("1", "Approved"), + ("3", "Pending"), + ("2", "Rejected"), + ], + db_index=True, + default="3", + max_length=1, + verbose_name="Status", + ), + ), + ( + "deleted_at", + models.DateTimeField(db_index=True, default=None, null=True), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "deleted_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_reviews", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="reviews", + to="logger.instance", + ), + ), + ( + "note", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="notes", + to="logger.note", + ), + ), ], ), migrations.AlterModelOptions( - name='mergedxform', + name="mergedxform", options={}, ), migrations.AlterModelOptions( - name='note', + name="note", options={}, ), migrations.AlterModelOptions( - name='project', - options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, + name="project", + options={ + "permissions": ( + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, ), migrations.AlterModelOptions( - name='xform', - options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + name="xform", + options={ + "ordering": ("pk",), + "permissions": ( + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(blank=True, default='', max_length=100), + model_name="attachment", + name="mimetype", + field=models.CharField(blank=True, default="", max_length=100), ), migrations.AlterField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.surveytype'), + model_name="instance", + name="survey_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="logger.surveytype" + ), ), migrations.AlterField( - model_name='instance', - name='user', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='osmdata', - name='field_name', - field=models.CharField(blank=True, default='', max_length=255), + model_name="osmdata", + name="field_name", + field=models.CharField(blank=True, default="", max_length=255), ), migrations.AlterField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default='way', max_length=10), + model_name="osmdata", + name="osm_type", + field=models.CharField(default="way", max_length=10), ), migrations.AlterField( - model_name='project', - name='tags', - field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + model_name="project", + name="tags", + field=taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + related_name="project_tags", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), ), migrations.AlterField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + model_name="widget", + name="order", + field=models.PositiveIntegerField( + db_index=True, editable=False, verbose_name="order" + ), ), migrations.AlterField( - model_name='widget', - name='widget_type', - field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), + model_name="widget", + name="widget_type", + field=models.CharField( + choices=[("charts", "Charts")], default="charts", max_length=25 + ), ), migrations.AlterField( - model_name='xform', - name='created_by', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + model_name="xform", + name="created_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='xform', - name='sms_id_string', - field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), + model_name="xform", + name="sms_id_string", + field=models.SlugField( + default="", editable=False, max_length=100, verbose_name="SMS ID" + ), ), migrations.AddField( - model_name='xform', - name='public_key', - field=models.TextField(blank=True, default='', null=True), + model_name="xform", + name="public_key", + field=models.TextField(blank=True, default="", null=True), ), migrations.AddField( - model_name='attachment', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), + model_name="attachment", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_attachments", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=36), ), migrations.RunPython(generate_uuid_if_missing), migrations.RunPython(regenerate_instance_json), migrations.CreateModel( - name='XFormVersion', + name="XFormVersion", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xls', models.FileField(upload_to='')), - ('version', models.CharField(max_length=100)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xml', models.TextField()), - ('json', models.TextField()), - ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.xform')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xls", models.FileField(upload_to="")), + ("version", models.CharField(max_length=100)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("xml", models.TextField()), + ("json", models.TextField()), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="logger.xform", + ), + ), ], options={ - 'unique_together': {('xform', 'version')}, + "unique_together": {("xform", "version")}, }, ), migrations.RunPython(create_initial_xform_version), diff --git a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py index 4ae94dca71..6293c1ed9a 100644 --- a/onadata/apps/logger/migrations/0002_auto_20150717_0048.py +++ b/onadata/apps/logger/migrations/0002_auto_20150717_0048.py @@ -7,25 +7,25 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_initial'), + ("logger", "0001_initial"), ] operations = [ migrations.AddField( - model_name='attachment', - name='date_created', + model_name="attachment", + name="date_created", field=models.DateTimeField(auto_now_add=True, null=True), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='date_modified', + model_name="attachment", + name="date_modified", field=models.DateTimeField(auto_now=True, null=True), preserve_default=True, ), migrations.AddField( - model_name='attachment', - name='deleted_at', + model_name="attachment", + name="deleted_at", field=models.DateTimeField(default=None, null=True), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0002_auto_20220425_0340.py b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py index 0762d5dffe..90d13c6afe 100644 --- a/onadata/apps/logger/migrations/0002_auto_20220425_0340.py +++ b/onadata/apps/logger/migrations/0002_auto_20220425_0340.py @@ -7,53 +7,57 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0001_pre-django-3-upgrade'), + ("logger", "0001_pre-django-3-upgrade"), ] operations = [ migrations.AlterField( - model_name='dataview', - name='columns', + model_name="dataview", + name="columns", field=models.JSONField(), ), migrations.AlterField( - model_name='dataview', - name='query', + model_name="dataview", + name="query", field=models.JSONField(blank=True, default=dict), ), migrations.AlterField( - model_name='instance', - name='json', + model_name="instance", + name="json", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='logger.xform'), + model_name="instance", + name="xform", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.xform", + ), ), migrations.AlterField( - model_name='osmdata', - name='tags', + model_name="osmdata", + name="tags", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='project', - name='metadata', + model_name="project", + name="metadata", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='widget', - name='metadata', + model_name="widget", + name="metadata", field=models.JSONField(blank=True, default=dict), ), migrations.AlterField( - model_name='xform', - name='json', + model_name="xform", + name="json", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='xform', - name='last_updated_at', + model_name="xform", + name="last_updated_at", field=models.DateTimeField(auto_now=True), ), ] diff --git a/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py index 195a3aecfb..b4fadc7c4a 100644 --- a/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py +++ b/onadata/apps/logger/migrations/0003_alter_instance_media_all_received.py @@ -6,13 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0002_auto_20220425_0340'), + ("logger", "0002_auto_20220425_0340"), ] operations = [ migrations.AlterField( - model_name='instance', - name='media_all_received', - field=models.BooleanField(default=True, null=True, verbose_name='Received All Media Attachemts'), + model_name="instance", + name="media_all_received", + field=models.BooleanField( + default=True, null=True, verbose_name="Received All Media Attachemts" + ), ), ] diff --git a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py index af36fdd965..3387d4ccf0 100644 --- a/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py +++ b/onadata/apps/logger/migrations/0003_dataview_instances_with_geopoints.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0002_auto_20150717_0048'), + ("logger", "0002_auto_20150717_0048"), ] operations = [ migrations.AddField( - model_name='dataview', - name='instances_with_geopoints', + model_name="dataview", + name="instances_with_geopoints", field=models.BooleanField(default=False), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py index 5949179146..86e737c21a 100644 --- a/onadata/apps/logger/migrations/0004_auto_20150910_0056.py +++ b/onadata/apps/logger/migrations/0004_auto_20150910_0056.py @@ -8,52 +8,77 @@ class Migration(migrations.Migration): dependencies = [ - ('auth', '0001_initial'), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0003_dataview_instances_with_geopoints'), + ("logger", "0003_dataview_instances_with_geopoints"), ] operations = [ migrations.CreateModel( - name='ProjectGroupObjectPermission', + name="ProjectGroupObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey( - to='logger.Project', on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ("group", models.ForeignKey(to="auth.Group", on_delete=models.CASCADE)), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.CreateModel( - name='ProjectUserObjectPermission', + name="ProjectUserObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.Project', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.Project", on_delete=models.CASCADE), + ), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), + ( + "user", + models.ForeignKey( + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='projectuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), + name="projectuserobjectpermission", + unique_together=set([("user", "permission", "content_object")]), ), migrations.AlterUniqueTogether( - name='projectgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), + name="projectgroupobjectpermission", + unique_together=set([("group", "permission", "content_object")]), ), ] diff --git a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py index db63190a51..4ea296e7e5 100644 --- a/onadata/apps/logger/migrations/0005_auto_20151015_0758.py +++ b/onadata/apps/logger/migrations/0005_auto_20151015_0758.py @@ -8,52 +8,77 @@ class Migration(migrations.Migration): dependencies = [ - ('auth', '0001_initial'), + ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0004_auto_20150910_0056'), + ("logger", "0004_auto_20150910_0056"), ] operations = [ migrations.CreateModel( - name='XFormGroupObjectPermission', + name="XFormGroupObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('group', models.ForeignKey(to='auth.Group', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), + ("group", models.ForeignKey(to="auth.Group", on_delete=models.CASCADE)), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.CreateModel( - name='XFormUserObjectPermission', + name="XFormUserObjectPermission", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, - auto_created=True, primary_key=True)), - ('content_object', models.ForeignKey(to='logger.XForm', - on_delete=models.CASCADE)), - ('permission', models.ForeignKey(to='auth.Permission', - on_delete=models.CASCADE)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, - on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "content_object", + models.ForeignKey(to="logger.XForm", on_delete=models.CASCADE), + ), + ( + "permission", + models.ForeignKey(to="auth.Permission", on_delete=models.CASCADE), + ), + ( + "user", + models.ForeignKey( + to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, bases=(models.Model,), ), migrations.AlterUniqueTogether( - name='xformuserobjectpermission', - unique_together=set([('user', 'permission', 'content_object')]), + name="xformuserobjectpermission", + unique_together=set([("user", "permission", "content_object")]), ), migrations.AlterUniqueTogether( - name='xformgroupobjectpermission', - unique_together=set([('group', 'permission', 'content_object')]), + name="xformgroupobjectpermission", + unique_together=set([("group", "permission", "content_object")]), ), ] diff --git a/onadata/apps/logger/migrations/0007_osmdata_field_name.py b/onadata/apps/logger/migrations/0007_osmdata_field_name.py index 7870df2c4e..5d579a1110 100644 --- a/onadata/apps/logger/migrations/0007_osmdata_field_name.py +++ b/onadata/apps/logger/migrations/0007_osmdata_field_name.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0006_auto_20151106_0130'), + ("logger", "0006_auto_20151106_0130"), ] operations = [ migrations.AddField( - model_name='osmdata', - name='field_name', - field=models.CharField(default=b'', max_length=255, blank=True), + model_name="osmdata", + name="field_name", + field=models.CharField(default=b"", max_length=255, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py index 7bde859ecf..e0f3354ea1 100644 --- a/onadata/apps/logger/migrations/0008_osmdata_osm_type.py +++ b/onadata/apps/logger/migrations/0008_osmdata_osm_type.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0007_osmdata_field_name'), + ("logger", "0007_osmdata_field_name"), ] operations = [ migrations.AddField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default=b'way', max_length=10), + model_name="osmdata", + name="osm_type", + field=models.CharField(default=b"way", max_length=10), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py index 96113b0393..2da2ae7381 100644 --- a/onadata/apps/logger/migrations/0009_auto_20151111_0438.py +++ b/onadata/apps/logger/migrations/0009_auto_20151111_0438.py @@ -7,12 +7,12 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0008_osmdata_osm_type'), + ("logger", "0008_osmdata_osm_type"), ] operations = [ migrations.AlterUniqueTogether( - name='osmdata', - unique_together=set([('instance', 'field_name')]), + name="osmdata", + unique_together=set([("instance", "field_name")]), ), ] diff --git a/onadata/apps/logger/migrations/0010_attachment_file_size.py b/onadata/apps/logger/migrations/0010_attachment_file_size.py index 63bcc1c617..71224e848c 100644 --- a/onadata/apps/logger/migrations/0010_attachment_file_size.py +++ b/onadata/apps/logger/migrations/0010_attachment_file_size.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0009_auto_20151111_0438'), + ("logger", "0009_auto_20151111_0438"), ] operations = [ migrations.AddField( - model_name='attachment', - name='file_size', + model_name="attachment", + name="file_size", field=models.PositiveIntegerField(default=0), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py index cee1b46ea5..14e0742af8 100644 --- a/onadata/apps/logger/migrations/0011_dataview_matches_parent.py +++ b/onadata/apps/logger/migrations/0011_dataview_matches_parent.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0010_attachment_file_size'), + ("logger", "0010_attachment_file_size"), ] operations = [ migrations.AddField( - model_name='dataview', - name='matches_parent', + model_name="dataview", + name="matches_parent", field=models.BooleanField(default=False), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py index ccbae70aaa..c82f2b402c 100644 --- a/onadata/apps/logger/migrations/0012_auto_20160114_0708.py +++ b/onadata/apps/logger/migrations/0012_auto_20160114_0708.py @@ -7,14 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0011_dataview_matches_parent'), + ("logger", "0011_dataview_matches_parent"), ] operations = [ migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(default=b'', max_length=100, blank=True), + model_name="attachment", + name="mimetype", + field=models.CharField(default=b"", max_length=100, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0013_note_created_by.py b/onadata/apps/logger/migrations/0013_note_created_by.py index eae6d40b75..8fbc71ff40 100644 --- a/onadata/apps/logger/migrations/0013_note_created_by.py +++ b/onadata/apps/logger/migrations/0013_note_created_by.py @@ -9,15 +9,19 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0012_auto_20160114_0708'), + ("logger", "0012_auto_20160114_0708"), ] operations = [ migrations.AddField( - model_name='note', - name='created_by', - field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, - null=True, on_delete=models.CASCADE), + model_name="note", + name="created_by", + field=models.ForeignKey( + blank=True, + to=settings.AUTH_USER_MODEL, + null=True, + on_delete=models.CASCADE, + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0014_note_instance_field.py b/onadata/apps/logger/migrations/0014_note_instance_field.py index 8db30ebdae..da25310973 100644 --- a/onadata/apps/logger/migrations/0014_note_instance_field.py +++ b/onadata/apps/logger/migrations/0014_note_instance_field.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0013_note_created_by'), + ("logger", "0013_note_created_by"), ] operations = [ migrations.AddField( - model_name='note', - name='instance_field', + model_name="note", + name="instance_field", field=models.TextField(null=True, blank=True), preserve_default=True, ), diff --git a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py index eb6cbcccd4..212a30a7e3 100644 --- a/onadata/apps/logger/migrations/0015_auto_20160222_0559.py +++ b/onadata/apps/logger/migrations/0015_auto_20160222_0559.py @@ -7,19 +7,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0014_note_instance_field'), + ("logger", "0014_note_instance_field"), ] operations = [ migrations.AlterModelOptions( - name='widget', - options={'ordering': ('order',)}, + name="widget", + options={"ordering": ("order",)}, ), migrations.AddField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(default=0, editable=False, - db_index=True), + model_name="widget", + name="order", + field=models.PositiveIntegerField(default=0, editable=False, db_index=True), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0016_widget_aggregation.py b/onadata/apps/logger/migrations/0016_widget_aggregation.py index d429a246ea..3f6aa8bda3 100644 --- a/onadata/apps/logger/migrations/0016_widget_aggregation.py +++ b/onadata/apps/logger/migrations/0016_widget_aggregation.py @@ -7,15 +7,14 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0015_auto_20160222_0559'), + ("logger", "0015_auto_20160222_0559"), ] operations = [ migrations.AddField( - model_name='widget', - name='aggregation', - field=models.CharField(default=None, max_length=255, null=True, - blank=True), + model_name="widget", + name="aggregation", + field=models.CharField(default=None, max_length=255, null=True, blank=True), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py index f6e71f7031..9d4fd9d38e 100644 --- a/onadata/apps/logger/migrations/0017_auto_20160224_0130.py +++ b/onadata/apps/logger/migrations/0017_auto_20160224_0130.py @@ -7,26 +7,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0016_widget_aggregation'), + ("logger", "0016_widget_aggregation"), ] operations = [ migrations.AlterField( - model_name='instance', - name='uuid', + model_name="instance", + name="uuid", field=models.CharField(max_length=249), preserve_default=True, ), migrations.AlterField( - model_name='instance', - name='xform', - field=models.ForeignKey(related_name='instances', default=-1, - to='logger.XForm', - on_delete=models.CASCADE), + model_name="instance", + name="xform", + field=models.ForeignKey( + related_name="instances", + default=-1, + to="logger.XForm", + on_delete=models.CASCADE, + ), preserve_default=False, ), migrations.AlterUniqueTogether( - name='instance', - unique_together=set([('xform', 'uuid')]), + name="instance", + unique_together=set([("xform", "uuid")]), ), ] diff --git a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py index 077a7dce24..bde2f67839 100644 --- a/onadata/apps/logger/migrations/0018_auto_20160301_0330.py +++ b/onadata/apps/logger/migrations/0018_auto_20160301_0330.py @@ -10,23 +10,24 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0017_auto_20160224_0130'), + ("logger", "0017_auto_20160224_0130"), ] operations = [ migrations.AddField( - model_name='instancehistory', - name='geom', + model_name="instancehistory", + name="geom", field=django.contrib.gis.db.models.fields.GeometryCollectionField( - srid=4326, null=True), + srid=4326, null=True + ), preserve_default=True, ), migrations.AddField( - model_name='instancehistory', - name='user', + model_name="instancehistory", + name="user", field=models.ForeignKey( - to=settings.AUTH_USER_MODEL, null=True, - on_delete=models.CASCADE), + to=settings.AUTH_USER_MODEL, null=True, on_delete=models.CASCADE + ), preserve_default=True, ), ] diff --git a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py index 9353329eae..141ec5f544 100644 --- a/onadata/apps/logger/migrations/0020_auto_20160408_0325.py +++ b/onadata/apps/logger/migrations/0020_auto_20160408_0325.py @@ -9,40 +9,42 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0019_auto_20160307_0256'), + ("logger", "0019_auto_20160307_0256"), ] operations = [ migrations.AlterField( - model_name='dataview', - name='columns', + model_name="dataview", + name="columns", field=django.contrib.postgres.fields.jsonb.JSONField(), ), migrations.AlterField( - model_name='dataview', - name='query', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), + model_name="dataview", + name="query", + field=django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), ), migrations.AlterField( - model_name='instance', - name='json', + model_name="instance", + name="json", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='osmdata', - name='tags', + model_name="osmdata", + name="tags", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='project', - name='metadata', + model_name="project", + name="metadata", field=django.contrib.postgres.fields.jsonb.JSONField(blank=True), ), migrations.AlterField( - model_name='widget', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, - default=dict), + model_name="widget", + name="metadata", + field=django.contrib.postgres.fields.jsonb.JSONField( + blank=True, default=dict + ), ), ] diff --git a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py index dbe27f3f87..56586ad9ae 100644 --- a/onadata/apps/logger/migrations/0021_auto_20160408_0919.py +++ b/onadata/apps/logger/migrations/0021_auto_20160408_0919.py @@ -9,13 +9,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0020_auto_20160408_0325'), + ("logger", "0020_auto_20160408_0325"), ] operations = [ migrations.AlterField( - model_name='project', - name='metadata', + model_name="project", + name="metadata", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), ] diff --git a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py index 8954428f43..aa92c8bfde 100644 --- a/onadata/apps/logger/migrations/0022_auto_20160418_0518.py +++ b/onadata/apps/logger/migrations/0022_auto_20160418_0518.py @@ -8,18 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0021_auto_20160408_0919'), + ("logger", "0021_auto_20160408_0919"), ] operations = [ migrations.AddField( - model_name='instance', - name='last_edited', + model_name="instance", + name="last_edited", field=models.DateTimeField(default=None, null=True), ), migrations.AddField( - model_name='instancehistory', - name='submission_date', + model_name="instancehistory", + name="submission_date", field=models.DateTimeField(default=None, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py index dc99e5bdb0..810289c740 100644 --- a/onadata/apps/logger/migrations/0023_auto_20160419_0403.py +++ b/onadata/apps/logger/migrations/0023_auto_20160419_0403.py @@ -8,19 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0022_auto_20160418_0518'), + ("logger", "0022_auto_20160418_0518"), ] operations = [ migrations.AlterField( - model_name='widget', - name='column', + model_name="widget", + name="column", field=models.CharField(max_length=255), ), migrations.AlterField( - model_name='widget', - name='group_by', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), + model_name="widget", + name="group_by", + field=models.CharField(blank=True, default=None, max_length=255, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py index 64a4ad8912..acecce134f 100644 --- a/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py +++ b/onadata/apps/logger/migrations/0024_xform_has_hxl_support.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0023_auto_20160419_0403'), + ("logger", "0023_auto_20160419_0403"), ] operations = [ migrations.AddField( - model_name='xform', - name='has_hxl_support', + model_name="xform", + name="has_hxl_support", field=models.BooleanField(default=False), ), ] diff --git a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py index f8000e7012..9d1ec9a4f9 100644 --- a/onadata/apps/logger/migrations/0025_xform_last_updated_at.py +++ b/onadata/apps/logger/migrations/0025_xform_last_updated_at.py @@ -10,17 +10,17 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0024_xform_has_hxl_support'), + ("logger", "0024_xform_has_hxl_support"), ] operations = [ migrations.AddField( - model_name='xform', - name='last_updated_at', + model_name="xform", + name="last_updated_at", field=models.DateTimeField( auto_now=True, - default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, - tzinfo=utc)), + default=datetime.datetime(2016, 8, 18, 12, 43, 30, 316792, tzinfo=utc), + ), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py index d29b823400..173122a4a5 100644 --- a/onadata/apps/logger/migrations/0026_auto_20160913_0239.py +++ b/onadata/apps/logger/migrations/0026_auto_20160913_0239.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0025_xform_last_updated_at'), + ("logger", "0025_xform_last_updated_at"), ] operations = [ migrations.AlterField( - model_name='osmdata', - name='osm_id', + model_name="osmdata", + name="osm_id", field=models.CharField(max_length=20), ), ] diff --git a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py index d253875e2c..559950558d 100644 --- a/onadata/apps/logger/migrations/0027_auto_20161201_0730.py +++ b/onadata/apps/logger/migrations/0027_auto_20161201_0730.py @@ -8,14 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0026_auto_20160913_0239'), + ("logger", "0026_auto_20160913_0239"), ] operations = [ migrations.AlterField( - model_name='widget', - name='title', - field=models.CharField(blank=True, default=None, max_length=255, - null=True), + model_name="widget", + name="title", + field=models.CharField(blank=True, default=None, max_length=255, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py index 39734a6f06..fbb3dcbfa3 100644 --- a/onadata/apps/logger/migrations/0028_auto_20170217_0502.py +++ b/onadata/apps/logger/migrations/0028_auto_20170217_0502.py @@ -9,44 +9,51 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0027_auto_20161201_0730'), + ("logger", "0027_auto_20161201_0730"), ] operations = [ migrations.AlterModelOptions( - name='project', - options={'permissions': ( - ('view_project', 'Can view project'), - ('add_project_xform', 'Can add xform to project'), - ('report_project_xform', - 'Can make submissions to the project'), - ('transfer_project', - 'Can transfer project to different owner'), - ('can_export_project_data', 'Can export data in project'), - ('view_project_all', 'Can view all associated data'), - ('view_project_data', 'Can view submitted data'))}, + name="project", + options={ + "permissions": ( + ("view_project", "Can view project"), + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('id_string',), - 'permissions': ( - ('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + "ordering": ("id_string",), + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), migrations.AlterField( - model_name='instance', - name='xform', + model_name="instance", + name="xform", field=models.ForeignKey( - default=328, on_delete=django.db.models.deletion.CASCADE, - related_name='instances', to='logger.XForm'), + default=328, + on_delete=django.db.models.deletion.CASCADE, + related_name="instances", + to="logger.XForm", + ), preserve_default=False, ), ] diff --git a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py index 07c68426c3..b059d64e03 100644 --- a/onadata/apps/logger/migrations/0028_auto_20170221_0838.py +++ b/onadata/apps/logger/migrations/0028_auto_20170221_0838.py @@ -9,26 +9,36 @@ class Migration(migrations.Migration): dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('logger', '0027_auto_20161201_0730'), + ("contenttypes", "0002_remove_content_type_name"), + ("logger", "0027_auto_20161201_0730"), ] operations = [ migrations.CreateModel( - name='OpenData', + name="OpenData", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, - serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('uuid', models.CharField(default='', max_length=32)), - ('object_id', models.PositiveIntegerField(blank=True, - null=True)), - ('active', models.BooleanField(default=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('content_type', models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to='contenttypes.ContentType')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("uuid", models.CharField(default="", max_length=32)), + ("object_id", models.PositiveIntegerField(blank=True, null=True)), + ("active", models.BooleanField(default=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.ContentType", + ), + ), ], ), ] diff --git a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py index 681b1890e9..c80adaf57b 100644 --- a/onadata/apps/logger/migrations/0029_auto_20170221_0908.py +++ b/onadata/apps/logger/migrations/0029_auto_20170221_0908.py @@ -9,14 +9,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0028_auto_20170221_0838'), + ("logger", "0028_auto_20170221_0838"), ] operations = [ migrations.AlterField( - model_name='opendata', - name='uuid', + model_name="opendata", + name="uuid", field=models.CharField( - default=onadata.apps.logger.models.open_data.get_uuid, - max_length=32), ), + default=onadata.apps.logger.models.open_data.get_uuid, max_length=32 + ), + ), ] diff --git a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py index a4e879564f..a764d7efab 100644 --- a/onadata/apps/logger/migrations/0030_auto_20170227_0137.py +++ b/onadata/apps/logger/migrations/0030_auto_20170227_0137.py @@ -9,15 +9,17 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0029_auto_20170221_0908'), + ("logger", "0029_auto_20170221_0908"), ] operations = [ migrations.AlterField( - model_name='opendata', - name='uuid', + model_name="opendata", + name="uuid", field=models.CharField( default=onadata.libs.utils.common_tools.get_uuid, max_length=32, - unique=True), ), + unique=True, + ), + ), ] diff --git a/onadata/apps/logger/migrations/0031_merge.py b/onadata/apps/logger/migrations/0031_merge.py index 3d24ace1f5..4a1ebd5bc6 100644 --- a/onadata/apps/logger/migrations/0031_merge.py +++ b/onadata/apps/logger/migrations/0031_merge.py @@ -8,9 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0028_auto_20170217_0502'), - ('logger', '0030_auto_20170227_0137'), + ("logger", "0028_auto_20170217_0502"), + ("logger", "0030_auto_20170227_0137"), ] - operations = [ - ] + operations = [] diff --git a/onadata/apps/logger/migrations/0032_project_deleted_at.py b/onadata/apps/logger/migrations/0032_project_deleted_at.py index 2fff7faf34..596027fdc2 100644 --- a/onadata/apps/logger/migrations/0032_project_deleted_at.py +++ b/onadata/apps/logger/migrations/0032_project_deleted_at.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0031_merge'), + ("logger", "0031_merge"), ] operations = [ migrations.AddField( - model_name='project', - name='deleted_at', + model_name="project", + name="deleted_at", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py index 3cc2db0cb1..e2f30634da 100644 --- a/onadata/apps/logger/migrations/0033_auto_20170705_0159.py +++ b/onadata/apps/logger/migrations/0033_auto_20170705_0159.py @@ -8,28 +8,30 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0032_project_deleted_at'), + ("logger", "0032_project_deleted_at"), ] operations = [ migrations.AlterModelOptions( - name='attachment', - options={'ordering': ('pk',)}, + name="attachment", + options={"ordering": ("pk",)}, ), migrations.AlterModelOptions( - name='xform', + name="xform", options={ - 'ordering': ('pk',), - 'permissions': - (('view_xform', 'Can view associated data'), - ('view_xform_all', 'Can view all associated data'), - ('view_xform_data', 'Can view submitted data'), - ('report_xform', 'Can make submissions to the form'), - ('move_xform', 'Can move form between projects'), - ('transfer_xform', 'Can transfer form ownership.'), - ('can_export_xform_data', 'Can export form data'), - ('delete_submission', 'Can delete submissions from form')), - 'verbose_name': 'XForm', - 'verbose_name_plural': 'XForms'}, + "ordering": ("pk",), + "permissions": ( + ("view_xform", "Can view associated data"), + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), ] diff --git a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py index 89841453ed..4d4f32ec01 100644 --- a/onadata/apps/logger/migrations/0034_auto_20170814_0432.py +++ b/onadata/apps/logger/migrations/0034_auto_20170814_0432.py @@ -8,32 +8,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0033_auto_20170705_0159'), + ("logger", "0033_auto_20170705_0159"), ] operations = [ migrations.AddField( - model_name='instance', - name='media_all_received', + model_name="instance", + name="media_all_received", field=models.NullBooleanField( - null=True, - verbose_name='Received All Media Attachemts' + null=True, verbose_name="Received All Media Attachemts" ), ), migrations.AddField( - model_name='instance', - name='media_count', + model_name="instance", + name="media_count", field=models.PositiveIntegerField( - null=True, - verbose_name='Received Media Attachments' + null=True, verbose_name="Received Media Attachments" ), ), migrations.AddField( - model_name='instance', - name='total_media', + model_name="instance", + name="total_media", field=models.PositiveIntegerField( - null=True, - verbose_name='Total Media Attachments' + null=True, verbose_name="Total Media Attachments" ), ), ] diff --git a/onadata/apps/logger/migrations/0034_mergedxform.py b/onadata/apps/logger/migrations/0034_mergedxform.py index 7d68dc8566..fdea7177d1 100644 --- a/onadata/apps/logger/migrations/0034_mergedxform.py +++ b/onadata/apps/logger/migrations/0034_mergedxform.py @@ -8,21 +8,32 @@ class Migration(migrations.Migration): - dependencies = [('logger', '0033_auto_20170705_0159'), ] + dependencies = [ + ("logger", "0033_auto_20170705_0159"), + ] operations = [ migrations.CreateModel( - name='MergedXForm', + name="MergedXForm", fields=[ - ('xform_ptr', models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to='logger.XForm')), - ('xforms', models.ManyToManyField( - related_name='mergedxform_ptr', to='logger.XForm')), + ( + "xform_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="logger.XForm", + ), + ), + ( + "xforms", + models.ManyToManyField( + related_name="mergedxform_ptr", to="logger.XForm" + ), + ), ], - bases=('logger.xform', ), ), + bases=("logger.xform",), + ), ] diff --git a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py index 6a4578e620..5861f6861e 100644 --- a/onadata/apps/logger/migrations/0035_auto_20170712_0529.py +++ b/onadata/apps/logger/migrations/0035_auto_20170712_0529.py @@ -7,13 +7,15 @@ class Migration(migrations.Migration): - dependencies = [('logger', '0034_mergedxform'), ] + dependencies = [ + ("logger", "0034_mergedxform"), + ] operations = [ migrations.AlterModelOptions( - name='mergedxform', + name="mergedxform", options={ - 'permissions': (('view_mergedxform', 'Can view associated data' - ), ) - }, ), + "permissions": (("view_mergedxform", "Can view associated data"),) + }, + ), ] diff --git a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py index 4705454762..2e80fa3c62 100644 --- a/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py +++ b/onadata/apps/logger/migrations/0036_xform_is_merged_dataset.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0035_auto_20170712_0529'), + ("logger", "0035_auto_20170712_0529"), ] operations = [ migrations.AddField( - model_name='xform', - name='is_merged_dataset', + model_name="xform", + name="is_merged_dataset", field=models.BooleanField(default=False), ), ] diff --git a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py index e6d9231e4c..8b385e0258 100644 --- a/onadata/apps/logger/migrations/0037_merge_20170825_0238.py +++ b/onadata/apps/logger/migrations/0037_merge_20170825_0238.py @@ -8,9 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0034_auto_20170814_0432'), - ('logger', '0036_xform_is_merged_dataset'), + ("logger", "0034_auto_20170814_0432"), + ("logger", "0036_xform_is_merged_dataset"), ] - operations = [ - ] + operations = [] diff --git a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py index 3ecb6b1668..028faf59cf 100644 --- a/onadata/apps/logger/migrations/0038_auto_20170828_1718.py +++ b/onadata/apps/logger/migrations/0038_auto_20170828_1718.py @@ -8,26 +8,29 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0037_merge_20170825_0238'), + ("logger", "0037_merge_20170825_0238"), ] operations = [ migrations.AlterField( - model_name='instance', - name='media_all_received', + model_name="instance", + name="media_all_received", field=models.NullBooleanField( - default=True, verbose_name='Received All Media Attachemts'), ), + default=True, verbose_name="Received All Media Attachemts" + ), + ), migrations.AlterField( - model_name='instance', - name='media_count', + model_name="instance", + name="media_count", field=models.PositiveIntegerField( - default=0, - null=True, - verbose_name='Received Media Attachments'), ), + default=0, null=True, verbose_name="Received Media Attachments" + ), + ), migrations.AlterField( - model_name='instance', - name='total_media', + model_name="instance", + name="total_media", field=models.PositiveIntegerField( - default=0, null=True, verbose_name='Total Media Attachments'), + default=0, null=True, verbose_name="Total Media Attachments" + ), ), ] diff --git a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py index 8ba7bffabb..bc2197a7ae 100644 --- a/onadata/apps/logger/migrations/0039_auto_20170909_2052.py +++ b/onadata/apps/logger/migrations/0039_auto_20170909_2052.py @@ -8,18 +8,18 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0038_auto_20170828_1718'), + ("logger", "0038_auto_20170828_1718"), ] operations = [ migrations.AddField( - model_name='instance', - name='checksum', + model_name="instance", + name="checksum", field=models.CharField(blank=True, max_length=64, null=True), ), migrations.AddField( - model_name='instancehistory', - name='checksum', + model_name="instancehistory", + name="checksum", field=models.CharField(blank=True, max_length=64, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py index 22c6225315..2c2d0b96e0 100644 --- a/onadata/apps/logger/migrations/0040_auto_20170912_1504.py +++ b/onadata/apps/logger/migrations/0040_auto_20170912_1504.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0039_auto_20170909_2052'), + ("logger", "0039_auto_20170909_2052"), ] operations = [ migrations.AlterField( - model_name='instance', - name='checksum', - field=models.CharField( - blank=True, db_index=True, max_length=64, null=True), ), + model_name="instance", + name="checksum", + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), ] diff --git a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py index fb082d79d0..a107435ce4 100644 --- a/onadata/apps/logger/migrations/0041_auto_20170912_1512.py +++ b/onadata/apps/logger/migrations/0041_auto_20170912_1512.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0040_auto_20170912_1504'), + ("logger", "0040_auto_20170912_1504"), ] operations = [ migrations.AlterField( - model_name='instance', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=249), + model_name="instance", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=249), ), ] diff --git a/onadata/apps/logger/migrations/0042_xform_hash.py b/onadata/apps/logger/migrations/0042_xform_hash.py index 8814600606..c6628b7b52 100644 --- a/onadata/apps/logger/migrations/0042_xform_hash.py +++ b/onadata/apps/logger/migrations/0042_xform_hash.py @@ -8,14 +8,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0041_auto_20170912_1512'), + ("logger", "0041_auto_20170912_1512"), ] operations = [ migrations.AddField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=32, - null=True, verbose_name='Hash'), + model_name="xform", + name="hash", + field=models.CharField( + blank=True, default=None, max_length=32, null=True, verbose_name="Hash" + ), ), ] diff --git a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py index a1da0b61ad..41e7118d4b 100644 --- a/onadata/apps/logger/migrations/0043_auto_20171010_0403.py +++ b/onadata/apps/logger/migrations/0043_auto_20171010_0403.py @@ -8,14 +8,15 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0042_xform_hash'), + ("logger", "0042_xform_hash"), ] operations = [ migrations.AlterField( - model_name='xform', - name='hash', - field=models.CharField(blank=True, default=None, max_length=36, - null=True, verbose_name='Hash'), + model_name="xform", + name="hash", + field=models.CharField( + blank=True, default=None, max_length=36, null=True, verbose_name="Hash" + ), ), ] diff --git a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py index 079550fd67..9823b90623 100644 --- a/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py +++ b/onadata/apps/logger/migrations/0044_xform_hash_sql_update.py @@ -7,12 +7,12 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0043_auto_20171010_0403'), + ("logger", "0043_auto_20171010_0403"), ] operations = [ migrations.RunSQL( "UPDATE logger_xform SET hash = CONCAT('md5:', MD5(XML)) WHERE hash IS NULL;", # noqa - migrations.RunSQL.noop + migrations.RunSQL.noop, ), ] diff --git a/onadata/apps/logger/migrations/0045_attachment_name.py b/onadata/apps/logger/migrations/0045_attachment_name.py index 33fcc5858a..a46dd78552 100644 --- a/onadata/apps/logger/migrations/0045_attachment_name.py +++ b/onadata/apps/logger/migrations/0045_attachment_name.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0044_xform_hash_sql_update'), + ("logger", "0044_xform_hash_sql_update"), ] operations = [ migrations.AddField( - model_name='attachment', - name='name', + model_name="attachment", + name="name", field=models.CharField(blank=True, max_length=100, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py index 982674e63a..f342b68690 100644 --- a/onadata/apps/logger/migrations/0046_auto_20180314_1618.py +++ b/onadata/apps/logger/migrations/0046_auto_20180314_1618.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0045_attachment_name'), + ("logger", "0045_attachment_name"), ] operations = [ migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(default="", max_length=36), ), ] diff --git a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py index 5342aa5bf8..1072becbb1 100644 --- a/onadata/apps/logger/migrations/0047_dataview_deleted_at.py +++ b/onadata/apps/logger/migrations/0047_dataview_deleted_at.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0046_auto_20180314_1618'), + ("logger", "0046_auto_20180314_1618"), ] operations = [ migrations.AddField( - model_name='dataview', - name='deleted_at', + model_name="dataview", + name="deleted_at", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py index 5822b374f8..bacacb0154 100644 --- a/onadata/apps/logger/migrations/0048_dataview_deleted_by.py +++ b/onadata/apps/logger/migrations/0048_dataview_deleted_by.py @@ -11,19 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0047_dataview_deleted_at'), + ("logger", "0047_dataview_deleted_at"), ] operations = [ migrations.AddField( - model_name='dataview', - name='deleted_by', + model_name="dataview", + name="deleted_by", field=models.ForeignKey( blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='dataview_deleted_by', - to=settings.AUTH_USER_MODEL), + related_name="dataview_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0049_xform_deleted_by.py b/onadata/apps/logger/migrations/0049_xform_deleted_by.py index 3c862e2118..a79ec9de0d 100644 --- a/onadata/apps/logger/migrations/0049_xform_deleted_by.py +++ b/onadata/apps/logger/migrations/0049_xform_deleted_by.py @@ -11,18 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0048_dataview_deleted_by'), + ("logger", "0048_dataview_deleted_by"), ] operations = [ migrations.AddField( - model_name='xform', - name='deleted_by', + model_name="xform", + name="deleted_by", field=models.ForeignKey( blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='xform_deleted_by', to=settings.AUTH_USER_MODEL), + related_name="xform_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0050_project_deleted_by.py b/onadata/apps/logger/migrations/0050_project_deleted_by.py index e96a4d29dc..5fd8889b11 100644 --- a/onadata/apps/logger/migrations/0050_project_deleted_by.py +++ b/onadata/apps/logger/migrations/0050_project_deleted_by.py @@ -11,17 +11,20 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0049_xform_deleted_by'), + ("logger", "0049_xform_deleted_by"), ] operations = [ migrations.AddField( - model_name='project', - name='deleted_by', + model_name="project", + name="deleted_by", field=models.ForeignKey( - blank=True, default=None, null=True, + blank=True, + default=None, + null=True, on_delete=django.db.models.deletion.SET_NULL, - related_name='project_deleted_by', - to=settings.AUTH_USER_MODEL), + related_name="project_deleted_by", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py index 24b4520e26..7e8067b2bd 100644 --- a/onadata/apps/logger/migrations/0051_auto_20180522_1118.py +++ b/onadata/apps/logger/migrations/0051_auto_20180522_1118.py @@ -15,15 +15,16 @@ def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 """ Recalculate all XForm hashes. """ - XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 - xforms = XForm.objects.filter(downloadable=True, - deleted_at__isnull=True).only('xml') + XForm = apps.get_model("logger", "XForm") # pylint: disable=C0103 + xforms = XForm.objects.filter(downloadable=True, deleted_at__isnull=True).only( + "xml" + ) count = xforms.count() counter = 0 for xform in queryset_iterator(xforms, 500): - xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest() - xform.save(update_fields=['hash']) + xform.hash = "md5:%s" % md5(xform.xml.encode("utf8")).hexdigest() + xform.save(update_fields=["hash"]) counter += 1 if counter % 500 == 0: print("Processed %d of %d forms." % (counter, count)) @@ -37,9 +38,7 @@ class Migration(migrations.Migration): """ dependencies = [ - ('logger', '0050_project_deleted_by'), + ("logger", "0050_project_deleted_by"), ] - operations = [ - migrations.RunPython(recalculate_xform_hash) - ] + operations = [migrations.RunPython(recalculate_xform_hash)] diff --git a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py index 364a048101..147a589ae5 100644 --- a/onadata/apps/logger/migrations/0052_auto_20180805_2233.py +++ b/onadata/apps/logger/migrations/0052_auto_20180805_2233.py @@ -14,17 +14,18 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0051_auto_20180522_1118'), + ("logger", "0051_auto_20180522_1118"), ] operations = [ migrations.AddField( - model_name='instance', - name='deleted_by', + model_name="instance", + name="deleted_by", field=models.ForeignKey( null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='deleted_instances', - to=settings.AUTH_USER_MODEL), + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0053_submissionreview.py b/onadata/apps/logger/migrations/0053_submissionreview.py index 555127c75b..000e768ce5 100644 --- a/onadata/apps/logger/migrations/0053_submissionreview.py +++ b/onadata/apps/logger/migrations/0053_submissionreview.py @@ -11,56 +11,77 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0052_auto_20180805_2233'), + ("logger", "0052_auto_20180805_2233"), ] operations = [ migrations.CreateModel( - name='SubmissionReview', + name="SubmissionReview", fields=[ - ('id', - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name='ID')), - ('status', - models.CharField( - choices=[('1', 'Approved'), ('3', 'Pending'), - ('2', 'Rejected')], - db_index=True, - default='3', - max_length=1, - verbose_name='Status')), - ('deleted_at', models.DateTimeField(default=None, null=True)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('created_by', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL)), - ('deleted_by', - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='deleted_reviews', - to=settings.AUTH_USER_MODEL)), - ('instance', - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='reviews', - to='logger.Instance')), - ('note', - models.ForeignKey( - blank=True, - default=None, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='notes', - to='logger.Note')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("1", "Approved"), + ("3", "Pending"), + ("2", "Rejected"), + ], + db_index=True, + default="3", + max_length=1, + verbose_name="Status", + ), + ), + ("deleted_at", models.DateTimeField(default=None, null=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "deleted_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_reviews", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="reviews", + to="logger.Instance", + ), + ), + ( + "note", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="notes", + to="logger.Note", + ), + ), ], ), ] diff --git a/onadata/apps/logger/migrations/0054_instance_has_a_review.py b/onadata/apps/logger/migrations/0054_instance_has_a_review.py index 739b52dff0..27ca162a9f 100644 --- a/onadata/apps/logger/migrations/0054_instance_has_a_review.py +++ b/onadata/apps/logger/migrations/0054_instance_has_a_review.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0053_submissionreview'), + ("logger", "0053_submissionreview"), ] operations = [ migrations.AddField( - model_name='instance', - name='has_a_review', - field=models.BooleanField(default=False, verbose_name='has_a_review'), + model_name="instance", + name="has_a_review", + field=models.BooleanField(default=False, verbose_name="has_a_review"), ), ] diff --git a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py index 19d80a1dbd..ee8710f90d 100644 --- a/onadata/apps/logger/migrations/0055_auto_20180904_0713.py +++ b/onadata/apps/logger/migrations/0055_auto_20180904_0713.py @@ -8,13 +8,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0054_instance_has_a_review'), + ("logger", "0054_instance_has_a_review"), ] operations = [ migrations.AlterField( - model_name='submissionreview', - name='deleted_at', + model_name="submissionreview", + name="deleted_at", field=models.DateTimeField(db_index=True, default=None, null=True), ), ] diff --git a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py index 2b2b51e7f9..a485c5c9ce 100644 --- a/onadata/apps/logger/migrations/0056_auto_20190125_0517.py +++ b/onadata/apps/logger/migrations/0056_auto_20190125_0517.py @@ -9,79 +9,130 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0055_auto_20180904_0713'), + ("logger", "0055_auto_20180904_0713"), ] operations = [ migrations.AlterModelOptions( - name='mergedxform', + name="mergedxform", options={}, ), migrations.AlterModelOptions( - name='note', + name="note", options={}, ), migrations.AlterModelOptions( - name='project', - options={'permissions': (('add_project_xform', 'Can add xform to project'), ('report_project_xform', 'Can make submissions to the project'), ('transfer_project', 'Can transfer project to different owner'), ('can_export_project_data', 'Can export data in project'), ('view_project_all', 'Can view all associated data'), ('view_project_data', 'Can view submitted data'))}, + name="project", + options={ + "permissions": ( + ("add_project_xform", "Can add xform to project"), + ("report_project_xform", "Can make submissions to the project"), + ("transfer_project", "Can transfer project to different owner"), + ("can_export_project_data", "Can export data in project"), + ("view_project_all", "Can view all associated data"), + ("view_project_data", "Can view submitted data"), + ) + }, ), migrations.AlterModelOptions( - name='xform', - options={'ordering': ('pk',), 'permissions': (('view_xform_all', 'Can view all associated data'), ('view_xform_data', 'Can view submitted data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('can_export_xform_data', 'Can export form data'), ('delete_submission', 'Can delete submissions from form')), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms'}, + name="xform", + options={ + "ordering": ("pk",), + "permissions": ( + ("view_xform_all", "Can view all associated data"), + ("view_xform_data", "Can view submitted data"), + ("report_xform", "Can make submissions to the form"), + ("move_xform", "Can move form between projects"), + ("transfer_xform", "Can transfer form ownership."), + ("can_export_xform_data", "Can export form data"), + ("delete_submission", "Can delete submissions from form"), + ), + "verbose_name": "XForm", + "verbose_name_plural": "XForms", + }, ), migrations.AlterField( - model_name='attachment', - name='mimetype', - field=models.CharField(blank=True, default='', max_length=100), + model_name="attachment", + name="mimetype", + field=models.CharField(blank=True, default="", max_length=100), ), migrations.AlterField( - model_name='instance', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='instance', - name='survey_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='logger.SurveyType'), + model_name="instance", + name="survey_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="logger.SurveyType" + ), ), migrations.AlterField( - model_name='instance', - name='user', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instances', to=settings.AUTH_USER_MODEL), + model_name="instance", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="instances", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='osmdata', - name='field_name', - field=models.CharField(blank=True, default='', max_length=255), + model_name="osmdata", + name="field_name", + field=models.CharField(blank=True, default="", max_length=255), ), migrations.AlterField( - model_name='osmdata', - name='osm_type', - field=models.CharField(default='way', max_length=10), + model_name="osmdata", + name="osm_type", + field=models.CharField(default="way", max_length=10), ), migrations.AlterField( - model_name='project', - name='tags', - field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', related_name='project_tags', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + model_name="project", + name="tags", + field=taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + related_name="project_tags", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), ), migrations.AlterField( - model_name='widget', - name='order', - field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + model_name="widget", + name="order", + field=models.PositiveIntegerField( + db_index=True, editable=False, verbose_name="order" + ), ), migrations.AlterField( - model_name='widget', - name='widget_type', - field=models.CharField(choices=[('charts', 'Charts')], default='charts', max_length=25), + model_name="widget", + name="widget_type", + field=models.CharField( + choices=[("charts", "Charts")], default="charts", max_length=25 + ), ), migrations.AlterField( - model_name='xform', - name='created_by', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + model_name="xform", + name="created_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='xform', - name='sms_id_string', - field=models.SlugField(default='', editable=False, max_length=100, verbose_name='SMS ID'), + model_name="xform", + name="sms_id_string", + field=models.SlugField( + default="", editable=False, max_length=100, verbose_name="SMS ID" + ), ), ] diff --git a/onadata/apps/logger/migrations/0057_xform_public_key.py b/onadata/apps/logger/migrations/0057_xform_public_key.py index 8d1e12848b..8c4054561e 100644 --- a/onadata/apps/logger/migrations/0057_xform_public_key.py +++ b/onadata/apps/logger/migrations/0057_xform_public_key.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0056_auto_20190125_0517'), + ("logger", "0056_auto_20190125_0517"), ] operations = [ migrations.AddField( - model_name='xform', - name='public_key', - field=models.TextField(default=''), + model_name="xform", + name="public_key", + field=models.TextField(default=""), ), ] diff --git a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py index 778a00a2e7..5be3a64f78 100644 --- a/onadata/apps/logger/migrations/0058_auto_20191211_0900.py +++ b/onadata/apps/logger/migrations/0058_auto_20191211_0900.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0057_xform_public_key'), + ("logger", "0057_xform_public_key"), ] operations = [ migrations.AlterField( - model_name='xform', - name='public_key', - field=models.TextField(blank=True, default='', null=True), + model_name="xform", + name="public_key", + field=models.TextField(blank=True, default="", null=True), ), ] diff --git a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py index 4a0cd2ac86..fbe84ad48c 100644 --- a/onadata/apps/logger/migrations/0059_attachment_deleted_by.py +++ b/onadata/apps/logger/migrations/0059_attachment_deleted_by.py @@ -9,13 +9,18 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0058_auto_20191211_0900'), + ("logger", "0058_auto_20191211_0900"), ] operations = [ migrations.AddField( - model_name='attachment', - name='deleted_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_attachments', to=settings.AUTH_USER_MODEL), + model_name="attachment", + name="deleted_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="deleted_attachments", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py index 4efeb9f57a..182082c2a1 100644 --- a/onadata/apps/logger/migrations/0060_auto_20200305_0357.py +++ b/onadata/apps/logger/migrations/0060_auto_20200305_0357.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('logger', '0059_attachment_deleted_by'), + ("logger", "0059_attachment_deleted_by"), ] operations = [ migrations.AlterField( - model_name='xform', - name='uuid', - field=models.CharField(db_index=True, default='', max_length=36), + model_name="xform", + name="uuid", + field=models.CharField(db_index=True, default="", max_length=36), ), ] diff --git a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py index 239cc02c2b..f67ecc8941 100644 --- a/onadata/apps/logger/migrations/0061_auto_20200713_0814.py +++ b/onadata/apps/logger/migrations/0061_auto_20200713_0814.py @@ -9,9 +9,9 @@ def generate_uuid_if_missing(apps, schema_editor): """ Generate uuids for XForms without them """ - XForm = apps.get_model('logger', 'XForm') + XForm = apps.get_model("logger", "XForm") - for xform in XForm.objects.filter(uuid=''): + for xform in XForm.objects.filter(uuid=""): xform.uuid = get_uuid() xform.save() @@ -19,8 +19,7 @@ def generate_uuid_if_missing(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0060_auto_20200305_0357'), + ("logger", "0060_auto_20200305_0357"), ] - operations = [ - migrations.RunPython(generate_uuid_if_missing)] + operations = [migrations.RunPython(generate_uuid_if_missing)] diff --git a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py index f545fdd69a..0dcfa1989d 100644 --- a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py +++ b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py @@ -10,9 +10,10 @@ def regenerate_instance_json(apps, schema_editor): Regenerate Instance JSON """ for inst in Instance.objects.filter( - deleted_at__isnull=True, - xform__downloadable=True, - xform__deleted_at__isnull=True): + deleted_at__isnull=True, + xform__downloadable=True, + xform__deleted_at__isnull=True, + ): inst.json = inst.get_full_dict(load_existing=False) inst.save() @@ -20,9 +21,7 @@ def regenerate_instance_json(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0061_auto_20200713_0814'), + ("logger", "0061_auto_20200713_0814"), ] - operations = [ - migrations.RunPython(regenerate_instance_json) - ] + operations = [migrations.RunPython(regenerate_instance_json)] diff --git a/onadata/apps/logger/migrations/0063_xformversion.py b/onadata/apps/logger/migrations/0063_xformversion.py index 064469a5d3..d19c0543b2 100644 --- a/onadata/apps/logger/migrations/0063_xformversion.py +++ b/onadata/apps/logger/migrations/0063_xformversion.py @@ -9,25 +9,47 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0062_auto_20210202_0248'), + ("logger", "0062_auto_20210202_0248"), ] operations = [ migrations.CreateModel( - name='XFormVersion', + name="XFormVersion", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('xls', models.FileField(upload_to='')), - ('version', models.CharField(max_length=100)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('date_modified', models.DateTimeField(auto_now=True)), - ('xml', models.TextField()), - ('json', models.TextField()), - ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ('xform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='logger.XForm')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("xls", models.FileField(upload_to="")), + ("version", models.CharField(max_length=100)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("date_modified", models.DateTimeField(auto_now=True)), + ("xml", models.TextField()), + ("json", models.TextField()), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "xform", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="logger.XForm", + ), + ), ], options={ - 'unique_together': {('xform', 'version')}, + "unique_together": {("xform", "version")}, }, ), ] diff --git a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py index 94b712f87e..59a0653dc4 100644 --- a/onadata/apps/logger/migrations/0064_auto_20210304_0314.py +++ b/onadata/apps/logger/migrations/0064_auto_20210304_0314.py @@ -11,10 +11,7 @@ def create_initial_xform_version(apps, schema_editor): Creates an XFormVersion object for an XForm that has no Version """ - queryset = XForm.objects.filter( - downloadable=True, - deleted_at__isnull=True - ) + queryset = XForm.objects.filter(downloadable=True, deleted_at__isnull=True) for xform in queryset.iterator(): if xform.version: create_xform_version(xform, xform.user) @@ -23,9 +20,7 @@ def create_initial_xform_version(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0063_xformversion'), + ("logger", "0063_xformversion"), ] - operations = [ - migrations.RunPython(create_initial_xform_version) - ] + operations = [migrations.RunPython(create_initial_xform_version)] diff --git a/onadata/apps/logger/models/attachment.py b/onadata/apps/logger/models/attachment.py index 9ea5520bfb..3e254fd222 100644 --- a/onadata/apps/logger/models/attachment.py +++ b/onadata/apps/logger/models/attachment.py @@ -14,23 +14,27 @@ def get_original_filename(filename): # before the extension. # this code trys to reverse this effect to derive the original name if filename: - parts = filename.split('_') + parts = filename.split("_") if len(parts) > 1: - ext_parts = parts[-1].split('.') + ext_parts = parts[-1].split(".") if len(ext_parts[0]) == 7 and len(ext_parts) == 2: ext = ext_parts[1] - return u'.'.join([u'_'.join(parts[:-1]), ext]) + return ".".join(["_".join(parts[:-1]), ext]) return filename def upload_to(instance, filename): - folder = "{}_{}".format(instance.instance.xform.id, - instance.instance.xform.id_string) + folder = "{}_{}".format( + instance.instance.xform.id, instance.instance.xform.id_string + ) return os.path.join( - instance.instance.xform.user.username, 'attachments', folder, - os.path.split(filename)[1]) + instance.instance.xform.user.username, + "attachments", + folder, + os.path.split(filename)[1], + ) class Attachment(models.Model): @@ -38,37 +42,37 @@ class Attachment(models.Model): Attachment model. """ - OSM = 'osm' + OSM = "osm" instance = models.ForeignKey( - 'logger.Instance', related_name="attachments", - on_delete=models.CASCADE) + "logger.Instance", related_name="attachments", on_delete=models.CASCADE + ) media_file = models.FileField(max_length=255, upload_to=upload_to) - mimetype = models.CharField( - max_length=100, null=False, blank=True, default='') - extension = models.CharField(max_length=10, null=False, blank=False, - default=u"non", db_index=True) + mimetype = models.CharField(max_length=100, null=False, blank=True, default="") + extension = models.CharField( + max_length=10, null=False, blank=False, default="non", db_index=True + ) date_created = models.DateTimeField(null=True, auto_now_add=True) date_modified = models.DateTimeField(null=True, auto_now=True) deleted_at = models.DateTimeField(null=True, default=None) file_size = models.PositiveIntegerField(default=0) name = models.CharField(max_length=100, null=True, blank=True) - deleted_by = models.ForeignKey(User, related_name='deleted_attachments', - null=True, on_delete=models.SET_NULL) + deleted_by = models.ForeignKey( + User, related_name="deleted_attachments", null=True, on_delete=models.SET_NULL + ) class Meta: - app_label = 'logger' - ordering = ("pk", ) + app_label = "logger" + ordering = ("pk",) def save(self, *args, **kwargs): - if self.media_file and self.mimetype == '': + if self.media_file and self.mimetype == "": # guess mimetype mimetype, encoding = mimetypes.guess_type(self.media_file.name) if mimetype: self.mimetype = mimetype if self.media_file and len(self.media_file.name) > 255: - raise ValueError( - "Length of the media file should be less or equal to 255") + raise ValueError("Length of the media file should be less or equal to 255") try: f_size = self.media_file.size @@ -82,8 +86,8 @@ def save(self, *args, **kwargs): @property def file_hash(self): if self.media_file.storage.exists(self.media_file.name): - return u'%s' % md5(self.media_file.read()).hexdigest() - return u'' + return "%s" % md5(self.media_file.read()).hexdigest() + return "" @property def filename(self): diff --git a/onadata/apps/logger/models/note.py b/onadata/apps/logger/models/note.py index c7ef49da83..549ea41b3b 100644 --- a/onadata/apps/logger/models/note.py +++ b/onadata/apps/logger/models/note.py @@ -12,12 +12,15 @@ class Note(models.Model): """ Note Model Class """ + note = models.TextField() instance = models.ForeignKey( - 'logger.Instance', related_name='notes', on_delete=models.CASCADE) + "logger.Instance", related_name="notes", on_delete=models.CASCADE + ) instance_field = models.TextField(null=True, blank=True) - created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, - blank=True, on_delete=models.CASCADE) + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.CASCADE + ) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -25,7 +28,8 @@ class Meta: """ Meta Options for Note Model """ - app_label = 'logger' + + app_label = "logger" def get_data(self): """ @@ -41,5 +45,5 @@ def get_data(self): "owner": owner, "note": self.note, "instance_field": self.instance_field, - "created_by": created_by_id + "created_by": created_by_id, } diff --git a/onadata/apps/logger/models/submission_review.py b/onadata/apps/logger/models/submission_review.py index 82ea73b94d..7a8205130e 100644 --- a/onadata/apps/logger/models/submission_review.py +++ b/onadata/apps/logger/models/submission_review.py @@ -26,40 +26,44 @@ class SubmissionReview(models.Model): SubmissionReview Model Class """ - APPROVED = '1' - REJECTED = '2' - PENDING = '3' + APPROVED = "1" + REJECTED = "2" + PENDING = "3" - STATUS_CHOICES = ((APPROVED, _('Approved')), (PENDING, _('Pending')), - (REJECTED, _('Rejected'))) + STATUS_CHOICES = ( + (APPROVED, _("Approved")), + (PENDING, _("Pending")), + (REJECTED, _("Rejected")), + ) instance = models.ForeignKey( - 'logger.Instance', related_name='reviews', on_delete=models.CASCADE) + "logger.Instance", related_name="reviews", on_delete=models.CASCADE + ) note = models.ForeignKey( - 'logger.Note', - related_name='notes', + "logger.Note", + related_name="notes", blank=True, null=True, default=None, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL, + ) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, null=True, blank=True, default=None, - on_delete=models.CASCADE) + on_delete=models.CASCADE, + ) status = models.CharField( - 'Status', - max_length=1, - choices=STATUS_CHOICES, - default=PENDING, - db_index=True) + "Status", max_length=1, choices=STATUS_CHOICES, default=PENDING, db_index=True + ) deleted_at = models.DateTimeField(null=True, default=None, db_index=True) deleted_by = models.ForeignKey( settings.AUTH_USER_MODEL, - related_name='deleted_reviews', + related_name="deleted_reviews", null=True, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL, + ) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) @@ -67,7 +71,8 @@ class Meta: """ Meta Options for SubmissionReview """ - app_label = 'logger' + + app_label = "logger" def get_note_text(self): """ @@ -90,5 +95,7 @@ def set_deleted(self, deleted_at=timezone.now(), user=None): post_save.connect( - update_instance_json_on_save, sender=SubmissionReview, - dispatch_uid='update_instance_json_on_save') + update_instance_json_on_save, + sender=SubmissionReview, + dispatch_uid="update_instance_json_on_save", +) diff --git a/onadata/apps/logger/models/xform_version.py b/onadata/apps/logger/models/xform_version.py index 3a6cc2c012..277a9bd0c7 100644 --- a/onadata/apps/logger/models/xform_version.py +++ b/onadata/apps/logger/models/xform_version.py @@ -13,15 +13,15 @@ class XFormVersion(models.Model): storage backend for utilization in the future when a user requires the previous XForm versions XML or JSON. """ + xform = models.ForeignKey( - 'logger.XForm', on_delete=models.CASCADE, related_name='versions') + "logger.XForm", on_delete=models.CASCADE, related_name="versions" + ) xls = models.FileField() version = models.CharField(max_length=100) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - created_by = models.ForeignKey( - 'auth.User', on_delete=models.SET_NULL, null=True - ) + created_by = models.ForeignKey("auth.User", on_delete=models.SET_NULL, null=True) xml = models.TextField() json = models.TextField() @@ -29,4 +29,4 @@ def __str__(self): return f"{self.xform.title}-{self.version}" class Meta: - unique_together = ['xform', 'version'] + unique_together = ["xform", "version"] diff --git a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py index 03ba6bb992..6bf2859be2 100644 --- a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py +++ b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py @@ -9,13 +9,15 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.import_tools import django_file -from onadata.apps.logger.management.commands.recover_deleted_attachments \ - import recover_deleted_attachments +from onadata.apps.logger.management.commands.recover_deleted_attachments import ( + recover_deleted_attachments, +) from onadata.libs.utils.logger_tools import create_instance class TestRecoverDeletedAttachments(TestBase): """TestRecoverDeletedAttachments Class""" + # pylint: disable=invalid-name def test_recovers_wrongly_deleted_attachments(self): """ @@ -40,20 +42,28 @@ def test_recovers_wrongly_deleted_attachments(self): 1300221157303.jpg """ - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 2) + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) attachment = instance.attachments.first() # Soft delete attachment @@ -62,13 +72,15 @@ def test_recovers_wrongly_deleted_attachments(self): attachment.save() self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 1) + instance.attachments.filter(deleted_at__isnull=True).count(), 1 + ) # Attempt recovery of attachment recover_deleted_attachments(form_id=instance.xform.id) self.assertEqual( - instance.attachments.filter(deleted_at__isnull=True).count(), 2) + instance.attachments.filter(deleted_at__isnull=True).count(), 2 + ) attachment.refresh_from_db() self.assertIsNone(attachment.deleted_at) self.assertIsNone(attachment.deleted_by) diff --git a/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py b/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py index ea27e836af..a0ac779b00 100644 --- a/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py +++ b/onadata/apps/logger/tests/management/commands/test_remove_columns_from_briefcase_data.py @@ -5,8 +5,9 @@ from io import BytesIO from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.management.commands.\ - remove_columns_from_briefcase_data import remove_columns_from_xml +from onadata.apps.logger.management.commands.remove_columns_from_briefcase_data import ( + remove_columns_from_xml, +) from onadata.libs.utils.logger_tools import create_instance @@ -38,15 +39,16 @@ def test_removes_correct_columns(self): """ instance = create_instance( - self.user.username, BytesIO(xml_string.strip().encode('utf-8')), - media_files=[] + self.user.username, + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[], ) expected_xml = ( f'<{id_string} id="{id_string}">' "uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7" "I love coding!" - f"") - modified_xml = remove_columns_from_xml( - instance.xml, ['first_name', 'photo']) + f"" + ) + modified_xml = remove_columns_from_xml(instance.xml, ["first_name", "photo"]) self.assertEqual(modified_xml, expected_xml) diff --git a/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py index d18cd1714d..f92036c396 100644 --- a/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py +++ b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py @@ -8,8 +8,9 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.import_tools import django_file -from onadata.apps.logger.management.commands.replace_form_id_root_node \ - import replace_form_id_with_correct_root_node +from onadata.apps.logger.management.commands.replace_form_id_root_node import ( + replace_form_id_with_correct_root_node, +) from onadata.libs.utils.logger_tools import create_instance @@ -39,22 +40,30 @@ def test_replaces_form_id_root_node(self): 1300221157303.jpg """ - media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' - '_2011_03_13.xml_2011-03-15_20-30-28/') + media_root = ( + f"{settings.PROJECT_ROOT}/apps/logger/tests/Health" + "_2011_03_13.xml_2011-03-15_20-30-28/" + ) image_media = django_file( - path=f'{media_root}1300221157303.jpg', field_name='image', - content_type='image/jpeg') + path=f"{media_root}1300221157303.jpg", + field_name="image", + content_type="image/jpeg", + ) file_media = django_file( - path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', - field_name='file', content_type='text/xml') + path=f"{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml", + field_name="file", + content_type="text/xml", + ) instance = create_instance( self.user.username, - BytesIO(xml_string.strip().encode('utf-8')), - media_files=[file_media, image_media]) + BytesIO(xml_string.strip().encode("utf-8")), + media_files=[file_media, image_media], + ) # Attempt replacement of root node name replace_form_id_with_correct_root_node( - inst_id=instance.id, root='data', commit=True) + inst_id=instance.id, root="data", commit=True + ) instance.refresh_from_db() expected_xml = f""" diff --git a/onadata/apps/logger/tests/models/test_attachment.py b/onadata/apps/logger/tests/models/test_attachment.py index d6437ab78f..ddb7be623c 100644 --- a/onadata/apps/logger/tests/models/test_attachment.py +++ b/onadata/apps/logger/tests/models/test_attachment.py @@ -9,115 +9,105 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import Attachment, Instance -from onadata.apps.logger.models.attachment import (get_original_filename, - upload_to) +from onadata.apps.logger.models.attachment import get_original_filename, upload_to from onadata.libs.utils.image_tools import image_url class TestAttachment(TestBase): - def setUp(self): super(self.__class__, self).setUp() self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create( - instance=self.instance, - media_file=File(open(media_file, 'rb'), media_file)) + instance=self.instance, media_file=File(open(media_file, "rb"), media_file) + ) def test_mimetype(self): - self.assertEqual(self.attachment.mimetype, 'image/jpeg') + self.assertEqual(self.attachment.mimetype, "image/jpeg") def test_create_attachment_with_mimetype_more_than_50(self): media_file = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', self.surveys[0], self.media_file) - media_file = File(open(media_file, 'rb'), media_file) + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.media_file, + ) + media_file = File(open(media_file, "rb"), media_file) with self.assertRaises(DataError): Attachment.objects.create( - instance=self.instance, - mimetype='a'*120, - media_file=media_file + instance=self.instance, mimetype="a" * 120, media_file=media_file ) pre_count = Attachment.objects.count() Attachment.objects.create( - instance=self.instance, - mimetype='a'*100, - media_file=media_file + instance=self.instance, mimetype="a" * 100, media_file=media_file ) self.assertEqual(pre_count + 1, Attachment.objects.count()) def test_create_attachment_with_media_file_length_more_the_100(self): with self.assertRaises(ValueError): - Attachment.objects.create( - instance=self.instance, - media_file='a'*300 - ) + Attachment.objects.create(instance=self.instance, media_file="a" * 300) pre_count = Attachment.objects.count() - Attachment.objects.create( - instance=self.instance, - media_file='a'*150 - ) + Attachment.objects.create(instance=self.instance, media_file="a" * 150) self.assertEqual(pre_count + 1, Attachment.objects.count()) def test_thumbnails(self): for attachment in Attachment.objects.filter(instance=self.instance): - url = image_url(attachment, 'small') - filename = attachment.media_file.name.replace('.jpg', '') - thumbnail = '%s-small.jpg' % filename - self.assertNotEqual( - url.find(thumbnail), -1) - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + url = image_url(attachment, "small") + filename = attachment.media_file.name.replace(".jpg", "") + thumbnail = "%s-small.jpg" % filename + self.assertNotEqual(url.find(thumbnail), -1) + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) default_storage.delete(thumbnail) def test_create_thumbnails_command(self): call_command("create_image_thumbnails") for attachment in Attachment.objects.filter(instance=self.instance): - filename = attachment.media_file.name.replace('.jpg', '') - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + filename = attachment.media_file.name.replace(".jpg", "") + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) check_datetime = timezone.now() # replace or regenerate thumbnails if they exist call_command("create_image_thumbnails", force=True) for attachment in Attachment.objects.filter(instance=self.instance): - filename = attachment.media_file.name.replace('.jpg', '') - for size in ['small', 'medium', 'large']: - thumbnail = '%s-%s.jpg' % (filename, size) - self.assertTrue( - default_storage.exists(thumbnail)) + filename = attachment.media_file.name.replace(".jpg", "") + for size in ["small", "medium", "large"]: + thumbnail = "%s-%s.jpg" % (filename, size) + self.assertTrue(default_storage.exists(thumbnail)) self.assertTrue( - default_storage.get_modified_time(thumbnail) > - check_datetime + default_storage.get_modified_time(thumbnail) > check_datetime ) default_storage.delete(thumbnail) def test_get_original_filename(self): self.assertEqual( - get_original_filename('submission.xml_K337n8u.enc'), - 'submission.xml.enc' + get_original_filename("submission.xml_K337n8u.enc"), "submission.xml.enc" ) self.assertEqual( - get_original_filename('submission.xml.enc'), - 'submission.xml.enc' + get_original_filename("submission.xml.enc"), "submission.xml.enc" ) self.assertEqual( - get_original_filename('submission_test.xml_K337n8u.enc'), - 'submission_test.xml.enc' + get_original_filename("submission_test.xml_K337n8u.enc"), + "submission_test.xml.enc", ) self.assertEqual( - get_original_filename('submission_random.enc'), - 'submission_random.enc' + get_original_filename("submission_random.enc"), "submission_random.enc" ) def test_upload_to(self): @@ -125,6 +115,9 @@ def test_upload_to(self): Test that upload to returns the correct path """ path = upload_to(self.attachment, self.attachment.filename) - self.assertEqual(path, - 'bob/attachments/{}_{}/1335783522563.jpg'.format( - self.xform.id, self.xform.id_string)) + self.assertEqual( + path, + "bob/attachments/{}_{}/1335783522563.jpg".format( + self.xform.id, self.xform.id_string + ), + ) diff --git a/onadata/apps/logger/tests/models/test_data_view.py b/onadata/apps/logger/tests/models/test_data_view.py index c8293ee158..573e0c4358 100644 --- a/onadata/apps/logger/tests/models/test_data_view.py +++ b/onadata/apps/logger/tests/models/test_data_view.py @@ -4,45 +4,20 @@ from django.db import connection from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ - TestAbstractViewSet -from onadata.apps.logger.models.data_view import ( - append_where_list, - DataView) +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.logger.models.data_view import append_where_list, DataView class TestDataView(TestBase): - def test_append_where_list(self): - json_str = 'json->>%s' - self.assertEqual( - append_where_list('<>', [], json_str), - [u'json->>%s <> %s'] - ) - self.assertEqual( - append_where_list('!=', [], json_str), - [u'json->>%s <> %s'] - ) - self.assertEqual( - append_where_list('=', [], json_str), - [u'json->>%s = %s'] - ) - self.assertEqual( - append_where_list('>', [], json_str), - [u'json->>%s > %s'] - ) - self.assertEqual( - append_where_list('<', [], json_str), - [u'json->>%s < %s'] - ) - self.assertEqual( - append_where_list('>=', [], json_str), - [u'json->>%s >= %s'] - ) - self.assertEqual( - append_where_list('<=', [], json_str), - [u'json->>%s <= %s'] - ) + json_str = "json->>%s" + self.assertEqual(append_where_list("<>", [], json_str), ["json->>%s <> %s"]) + self.assertEqual(append_where_list("!=", [], json_str), ["json->>%s <> %s"]) + self.assertEqual(append_where_list("=", [], json_str), ["json->>%s = %s"]) + self.assertEqual(append_where_list(">", [], json_str), ["json->>%s > %s"]) + self.assertEqual(append_where_list("<", [], json_str), ["json->>%s < %s"]) + self.assertEqual(append_where_list(">=", [], json_str), ["json->>%s >= %s"]) + self.assertEqual(append_where_list("<=", [], json_str), ["json->>%s <= %s"]) class TestIntegratedDataView(TestAbstractViewSet): @@ -62,26 +37,36 @@ def setUp(self): def _setup_dataview(self): xlsform_path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", "fixtures", - "tutorial.xlsx") + settings.PROJECT_ROOT, "libs", "tests", "utils", "fixtures", "tutorial.xlsx" + ) self._publish_xls_form_to_project(xlsform_path=xlsform_path) for x in range(1, 9): path = os.path.join( - settings.PROJECT_ROOT, 'libs', 'tests', "utils", 'fixtures', - 'tutorial', 'instances', 'uuid{}'.format(x), 'submission.xml') + settings.PROJECT_ROOT, + "libs", + "tests", + "utils", + "fixtures", + "tutorial", + "instances", + "uuid{}".format(x), + "submission.xml", + ) self._make_submission(path) x += 1 self._create_dataview() def test_generate_query_string_for_data_without_filter(self): - expected_sql = "SELECT json FROM "\ - "logger_instance WHERE xform_id = %s AND "\ - "CAST(json->>%s AS INT) > %s AND "\ - "CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY id" + expected_sql = ( + "SELECT json FROM " + "logger_instance WHERE xform_id = %s AND " + "CAST(json->>%s AS INT) > %s AND " + "CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY id" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -89,7 +74,8 @@ def test_generate_query_string_for_data_without_filter(self): self.limit, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) self.assertEqual(sql, expected_sql) self.assertEqual(len(columns), 8) @@ -101,10 +87,12 @@ def test_generate_query_string_for_data_without_filter(self): def test_generate_query_string_for_data_with_limit_filter(self): limit_filter = 1 - expected_sql = "SELECT json FROM logger_instance"\ - " WHERE xform_id = %s AND CAST(json->>%s AS INT) > %s"\ - " AND CAST(json->>%s AS INT) < %s AND deleted_at "\ - "IS NULL ORDER BY id LIMIT %s" + expected_sql = ( + "SELECT json FROM logger_instance" + " WHERE xform_id = %s AND CAST(json->>%s AS INT) > %s" + " AND CAST(json->>%s AS INT) < %s AND deleted_at " + "IS NULL ORDER BY id LIMIT %s" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -112,23 +100,26 @@ def test_generate_query_string_for_data_with_limit_filter(self): limit_filter, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertEqual(len(records), limit_filter) def test_generate_query_string_for_data_with_start_index_filter(self): start_index = 2 - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL "\ - "ORDER BY id OFFSET %s" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL " + "ORDER BY id OFFSET %s" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -136,26 +127,29 @@ def test_generate_query_string_for_data_with_start_index_filter(self): self.limit, self.last_submission_time, self.all_data, - self.sort) + self.sort, + ) self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertEqual(len(records), 1) - self.assertIn('name', records[0]) - self.assertIn('age', records[0]) - self.assertIn('gender', records[0]) - self.assertNotIn('pizza_type', records[0]) + self.assertIn("name", records[0]) + self.assertIn("age", records[0]) + self.assertIn("gender", records[0]) + self.assertNotIn("pizza_type", records[0]) def test_generate_query_string_for_data_with_sort_column_asc(self): sort = '{"age":1}' - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY json->>%s ASC" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY json->>%s ASC" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -163,23 +157,26 @@ def test_generate_query_string_for_data_with_sort_column_asc(self): self.limit, self.last_submission_time, self.all_data, - sort) + sort, + ) self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertTrue(self.is_sorted_asc([r.get("age") for r in records])) def test_generate_query_string_for_data_with_sort_column_desc(self): sort = '{"age": -1}' - expected_sql = "SELECT json FROM logger_instance WHERE"\ - " xform_id = %s AND CAST(json->>%s AS INT) > %s AND"\ - " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL"\ - " ORDER BY json->>%s DESC" + expected_sql = ( + "SELECT json FROM logger_instance WHERE" + " xform_id = %s AND CAST(json->>%s AS INT) > %s AND" + " CAST(json->>%s AS INT) < %s AND deleted_at IS NULL" + " ORDER BY json->>%s DESC" + ) (sql, columns, params) = DataView.generate_query_string( self.data_view, @@ -187,13 +184,14 @@ def test_generate_query_string_for_data_with_sort_column_desc(self): self.limit, self.last_submission_time, self.all_data, - sort) + sort, + ) self.assertEqual(sql, expected_sql) - records = [record for record in DataView.query_iterator(sql, - columns, - params, - self.count)] + records = [ + record + for record in DataView.query_iterator(sql, columns, params, self.count) + ] self.assertTrue(self.is_sorted_desc([r.get("age") for r in records])) diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 99bfb6e7c7..06741cc3a5 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -9,18 +9,23 @@ from onadata.apps.logger.models import XForm, Instance, SubmissionReview from onadata.apps.logger.models.instance import ( - get_id_string_from_xml_str, numeric_checker) + get_id_string_from_xml_str, + numeric_checker, +) from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.viewer.models.parsed_instance import ( - ParsedInstance, query_data) -from onadata.libs.serializers.submission_review_serializer import \ - SubmissionReviewSerializer -from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME, \ - XFORM_ID_STRING, SUBMITTED_BY +from onadata.apps.viewer.models.parsed_instance import ParsedInstance, query_data +from onadata.libs.serializers.submission_review_serializer import ( + SubmissionReviewSerializer, +) +from onadata.libs.utils.common_tags import ( + MONGO_STRFTIME, + SUBMISSION_TIME, + XFORM_ID_STRING, + SUBMITTED_BY, +) class TestInstance(TestBase): - def setUp(self): super(self.__class__, self).setUp() @@ -31,7 +36,7 @@ def test_stores_json(self): for instance in instances: self.assertNotEqual(instance.json, {}) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_json_assigns_attributes(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._publish_transportation_form_and_submit_instance() @@ -40,12 +45,13 @@ def test_json_assigns_attributes(self, mock_time): instances = Instance.objects.all() for instance in instances: - self.assertEqual(instance.json[SUBMISSION_TIME], - mock_time.return_value.strftime(MONGO_STRFTIME)) - self.assertEqual(instance.json[XFORM_ID_STRING], - xform_id_string) + self.assertEqual( + instance.json[SUBMISSION_TIME], + mock_time.return_value.strftime(MONGO_STRFTIME), + ) + self.assertEqual(instance.json[XFORM_ID_STRING], xform_id_string) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_json_stores_user_attribute(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._publish_transportation_form() @@ -56,8 +62,13 @@ def test_json_stores_user_attribute(self, mock_time): # submit instance with a request user path = os.path.join( - self.this_directory, 'fixtures', 'transportation', 'instances', - self.surveys[0], self.surveys[0] + '.xml') + self.this_directory, + "fixtures", + "transportation", + "instances", + self.surveys[0], + self.surveys[0] + ".xml", + ) auth = DigestAuth(self.login_username, self.login_password) self._make_submission(path, auth=auth) @@ -66,19 +77,21 @@ def test_json_stores_user_attribute(self, mock_time): self.assertTrue(len(instances) > 0) for instance in instances: - self.assertEqual(instance.json[SUBMITTED_BY], 'bob') + self.assertEqual(instance.json[SUBMITTED_BY], "bob") # check that the parsed instance's to_dict_for_mongo also contains # the _user key, which is what's used by the JSON REST service pi = ParsedInstance.objects.get(instance=instance) - self.assertEqual(pi.to_dict_for_mongo()[SUBMITTED_BY], 'bob') + self.assertEqual(pi.to_dict_for_mongo()[SUBMITTED_BY], "bob") def test_json_time_match_submission_time(self): self._publish_transportation_form_and_submit_instance() instances = Instance.objects.all() for instance in instances: - self.assertEqual(instance.json[SUBMISSION_TIME], - instance.date_created.strftime(MONGO_STRFTIME)) + self.assertEqual( + instance.json[SUBMISSION_TIME], + instance.date_created.strftime(MONGO_STRFTIME), + ) def test_set_instances_with_geopoints_on_submission_false(self): self._publish_transportation_form() @@ -96,8 +109,7 @@ def test_instances_with_geopoints_in_repeats(self): self.assertFalse(self.xform.instances_with_geopoints) - instance_path = self._fixture_path("gps", - "gps_in_repeats_submission.xml") + instance_path = self._fixture_path("gps", "gps_in_repeats_submission.xml") self._make_submission(instance_path) xform = XForm.objects.get(pk=self.xform.pk) @@ -114,17 +126,17 @@ def test_set_instances_with_geopoints_on_submission_true(self): self.assertTrue(xform.instances_with_geopoints) - @patch('onadata.apps.logger.models.instance.get_values_matching_key') + @patch("onadata.apps.logger.models.instance.get_values_matching_key") def test_instances_with_malformed_geopoints_dont_trigger_value_error( - self, mock_get_values_matching_key): - mock_get_values_matching_key.return_value = '40.81101715564728' + self, mock_get_values_matching_key + ): + mock_get_values_matching_key.return_value = "40.81101715564728" xls_path = self._fixture_path("gps", "gps.xlsx") self._publish_xls_file_and_set_xform(xls_path) self.assertFalse(self.xform.instances_with_geopoints) - path = self._fixture_path( - 'gps', 'instances', 'gps_1980-01-23_20-52-08.xml') + path = self._fixture_path("gps", "instances", "gps_1980-01-23_20-52-08.xml") self._make_submission(path) xform = XForm.objects.get(pk=self.xform.pk) self.assertFalse(xform.instances_with_geopoints) @@ -141,28 +153,25 @@ def test_get_id_string_from_xml_str(self): """ id_string = get_id_string_from_xml_str(submission) - self.assertEqual(id_string, 'id_string') + self.assertEqual(id_string, "id_string") def test_query_data_sort(self): self._publish_transportation_form() self._make_submissions() - latest = Instance.objects.filter(xform=self.xform).latest('pk').pk + latest = Instance.objects.filter(xform=self.xform).latest("pk").pk oldest = Instance.objects.filter(xform=self.xform).first().pk - data = [i.get('_id') for i in query_data( - self.xform, sort='-_id')] + data = [i.get("_id") for i in query_data(self.xform, sort="-_id")] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field - data = [i.get('_id') for i in query_data( - self.xform, sort='{"_id": "-1"}')] + data = [i.get("_id") for i in query_data(self.xform, sort='{"_id": "-1"}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field - data = [i.get('_id') for i in query_data( - self.xform, sort='{"_id": -1}')] + data = [i.get("_id") for i in query_data(self.xform, sort='{"_id": -1}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) @@ -171,45 +180,63 @@ def test_query_filter_by_integer(self): self._make_submissions() oldest = Instance.objects.filter(xform=self.xform).first().pk - data = [i.get('_id') for i in query_data( - self.xform, query='[{"_id": %s}]' % (oldest))] + data = [ + i.get("_id") + for i in query_data(self.xform, query='[{"_id": %s}]' % (oldest)) + ] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # with fields - data = [i.get('_id') for i in query_data( - self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')] + data = [ + i.get("_id") + for i in query_data( + self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]' + ) + ] self.assertEqual(len(data), 1) self.assertEqual(data, [str(oldest)]) # mongo $gt - data = [i.get('_id') for i in query_data( - self.xform, query='{"_id": {"$gt": %s}}' % (oldest), - fields='["_id"]')] + data = [ + i.get("_id") + for i in query_data( + self.xform, query='{"_id": {"$gt": %s}}' % (oldest), fields='["_id"]' + ) + ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) - @patch('onadata.apps.logger.models.instance.submission_time') + @patch("onadata.apps.logger.models.instance.submission_time") def test_query_filter_by_datetime_field(self, mock_time): self._publish_transportation_form() now = datetime(2014, 1, 1, tzinfo=utc) - times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2), - now + timedelta(seconds=3)] + times = [ + now, + now + timedelta(seconds=1), + now + timedelta(seconds=2), + now + timedelta(seconds=3), + ] mock_time.side_effect = times self._make_submissions() atime = None - for i in self.xform.instances.all().order_by('-pk'): + for i in self.xform.instances.all().order_by("-pk"): i.date_created = times.pop() i.save() if atime is None: atime = i.date_created.strftime(MONGO_STRFTIME) # mongo $gt - data = [i.get('_submission_time') for i in query_data( - self.xform, query='{"_submission_time": {"$lt": "%s"}}' % (atime), - fields='["_submission_time"]')] + data = [ + i.get("_submission_time") + for i in query_data( + self.xform, + query='{"_submission_time": {"$lt": "%s"}}' % (atime), + fields='["_submission_time"]', + ) + ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) self.assertNotIn(atime, data) @@ -224,29 +251,26 @@ def test_instance_json_updated_on_review(self): """ self._publish_transportation_form_and_submit_instance() instance = Instance.objects.first() - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_comment', instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) self.assertFalse(instance.has_a_review) - data = { - "instance": instance.id, - "status": SubmissionReview.APPROVED - } + data = {"instance": instance.id, "status": SubmissionReview.APPROVED} request = HttpRequest() request.user = self.user - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() instance_review = instance.get_latest_review() - self.assertNotIn(u'_review_comment', instance.json.keys()) - self.assertIn(u'_review_status', instance.json.keys()) - self.assertIn(u'_review_date', instance.json.keys()) - self.assertEqual(SubmissionReview.APPROVED, - instance.json[u'_review_status']) + self.assertNotIn("_review_comment", instance.json.keys()) + self.assertIn("_review_status", instance.json.keys()) + self.assertIn("_review_date", instance.json.keys()) + self.assertEqual(SubmissionReview.APPROVED, instance.json["_review_status"]) self.assertEqual(SubmissionReview.APPROVED, instance_review.status) comment = instance_review.get_note_text() self.assertEqual(None, comment) @@ -255,21 +279,21 @@ def test_instance_json_updated_on_review(self): data = { "instance": instance.id, "note": "Hey There", - "status": SubmissionReview.APPROVED + "status": SubmissionReview.APPROVED, } - serializer_instance = SubmissionReviewSerializer(data=data, context={ - "request": request}) + serializer_instance = SubmissionReviewSerializer( + data=data, context={"request": request} + ) serializer_instance.is_valid() serializer_instance.save() instance.refresh_from_db() instance_review = instance.get_latest_review() - self.assertIn(u'_review_comment', instance.json.keys()) - self.assertIn(u'_review_status', instance.json.keys()) - self.assertIn(u'_review_date', instance.json.keys()) - self.assertEqual(SubmissionReview.APPROVED, - instance.json[u'_review_status']) + self.assertIn("_review_comment", instance.json.keys()) + self.assertIn("_review_status", instance.json.keys()) + self.assertIn("_review_date", instance.json.keys()) + self.assertEqual(SubmissionReview.APPROVED, instance.json["_review_status"]) self.assertEqual(SubmissionReview.APPROVED, instance_review.status) comment = instance_review.get_note_text() self.assertEqual("Hey There", comment) @@ -283,8 +307,8 @@ def test_retrieve_non_existent_submission_review(self): self._publish_transportation_form_and_submit_instance() instance = Instance.objects.first() - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_comment', instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) self.assertFalse(instance.has_a_review) # Update instance has_a_review field @@ -297,9 +321,9 @@ def test_retrieve_non_existent_submission_review(self): self.assertIsNone(instance.get_latest_review()) # Test instance json is not updated - self.assertNotIn(u'_review_comment', instance.json.keys()) - self.assertNotIn(u'_review_status', instance.json.keys()) - self.assertNotIn(u'_review_date', instance.json.keys()) + self.assertNotIn("_review_comment", instance.json.keys()) + self.assertNotIn("_review_status", instance.json.keys()) + self.assertNotIn("_review_date", instance.json.keys()) def test_numeric_checker_with_negative_integer_values(self): # Evaluate negative integer values @@ -318,7 +342,7 @@ def test_numeric_checker_with_negative_integer_values(self): self.assertEqual(result, 36.23) # Evaluate nan values - string_value = float('NaN') + string_value = float("NaN") result = numeric_checker(string_value) self.assertEqual(result, 0) diff --git a/onadata/apps/logger/tests/models/test_note.py b/onadata/apps/logger/tests/models/test_note.py index b6c44412a5..13318d4e90 100644 --- a/onadata/apps/logger/tests/models/test_note.py +++ b/onadata/apps/logger/tests/models/test_note.py @@ -24,5 +24,5 @@ def test_no_created_by(self): ) note.save() note_data = note.get_data() - self.assertEqual(note_data['owner'], "") - self.assertEqual(note_data['created_by'], "") + self.assertEqual(note_data["owner"], "") + self.assertEqual(note_data["created_by"], "") diff --git a/onadata/apps/logger/tests/models/test_submission_review.py b/onadata/apps/logger/tests/models/test_submission_review.py index 00f36a6ec0..ec40421470 100644 --- a/onadata/apps/logger/tests/models/test_submission_review.py +++ b/onadata/apps/logger/tests/models/test_submission_review.py @@ -24,7 +24,7 @@ def test_note_text_property_method(self): instance = Instance.objects.first() note = Note( instance=instance, - note='Hey there', + note="Hey there", instance_field="", ) diff --git a/onadata/apps/logger/tests/test_backup_tools.py b/onadata/apps/logger/tests/test_backup_tools.py index e9509e9fb0..eeebcd9679 100644 --- a/onadata/apps/logger/tests/test_backup_tools.py +++ b/onadata/apps/logger/tests/test_backup_tools.py @@ -9,8 +9,11 @@ from onadata.apps.logger.models import Instance from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.import_tools import create_instance -from onadata.libs.utils.backup_tools import _date_created_from_filename,\ - create_zip_backup, restore_backup_from_zip +from onadata.libs.utils.backup_tools import ( + _date_created_from_filename, + create_zip_backup, + restore_backup_from_zip, +) class TestBackupTools(TestBase): @@ -19,8 +22,13 @@ def setUp(self): self._publish_xls_file_and_set_xform( os.path.join( settings.PROJECT_ROOT, - "apps", "logger", "fixtures", "test_forms", - "tutorial.xlsx")) + "apps", + "logger", + "fixtures", + "test_forms", + "tutorial.xlsx", + ) + ) def test_date_created_override(self): """ @@ -28,19 +36,30 @@ def test_date_created_override(self): will set our date as the date_created """ xml_file_path = os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml") + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ) xml_file = django_file( - xml_file_path, field_name="xml_file", content_type="text/xml") + xml_file_path, field_name="xml_file", content_type="text/xml" + ) media_files = [] - date_created = datetime.strptime("2013-01-01 12:00:00", - "%Y-%m-%d %H:%M:%S") + date_created = datetime.strptime("2013-01-01 12:00:00", "%Y-%m-%d %H:%M:%S") instance = create_instance( - self.user.username, xml_file, media_files, - date_created_override=date_created) + self.user.username, + xml_file, + media_files, + date_created_override=date_created, + ) self.assertIsNotNone(instance) - self.assertEqual(instance.date_created.strftime("%Y-%m-%d %H:%M:%S"), - date_created.strftime("%Y-%m-%d %H:%M:%S")) + self.assertEqual( + instance.date_created.strftime("%Y-%m-%d %H:%M:%S"), + date_created.strftime("%Y-%m-%d %H:%M:%S"), + ) def test_date_created_from_filename(self): date_str = "2012-01-02-12-35-48" @@ -55,17 +74,14 @@ def test_date_created_from_filename(self): def test_backup_then_restore_from_zip(self): self._publish_transportation_form() - initial_instance_count = Instance.objects.filter( - xform=self.xform).count() + initial_instance_count = Instance.objects.filter(xform=self.xform).count() # make submissions for i in range(len(self.surveys)): self._submit_transport_instance(i) - instance_count = Instance.objects.filter( - xform=self.xform).count() - self.assertEqual( - instance_count, initial_instance_count + len(self.surveys)) + instance_count = Instance.objects.filter(xform=self.xform).count() + self.assertEqual(instance_count, initial_instance_count + len(self.surveys)) # make a backup temp_dir = tempfile.mkdtemp() @@ -77,17 +93,14 @@ def test_backup_then_restore_from_zip(self): for instance in Instance.objects.filter(xform=self.xform): instance.delete() - instance_count = Instance.objects.filter( - xform=self.xform).count() + instance_count = Instance.objects.filter(xform=self.xform).count() # remove temp dir tree self.assertEqual(instance_count, 0) # restore instances self.assertTrue(os.path.exists(zip_file.name)) - restore_backup_from_zip( - zip_file.name, self.user.username) - instance_count = Instance.objects.filter( - xform=self.xform).count() + restore_backup_from_zip(zip_file.name, self.user.username) + instance_count = Instance.objects.filter(xform=self.xform).count() # remove temp dir tree self.assertEqual(instance_count, len(self.surveys)) shutil.rmtree(temp_dir) diff --git a/onadata/apps/logger/tests/test_briefcase_api.py b/onadata/apps/logger/tests/test_briefcase_api.py index 1dca42662d..5077093359 100644 --- a/onadata/apps/logger/tests/test_briefcase_api.py +++ b/onadata/apps/logger/tests/test_briefcase_api.py @@ -21,33 +21,35 @@ def ordered_instances(xform): - return Instance.objects.filter(xform=xform).order_by('id') + return Instance.objects.filter(xform=xform).order_by("id") class TestBriefcaseAPI(TestBase): - def setUp(self): super(TestBase, self).setUp() self.factory = APIRequestFactory() self._create_user_and_login() self._logout() self.form_def_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation.xml') + self.this_directory, "fixtures", "transportation", "transportation.xml" + ) self._submission_list_url = reverse( - 'view-submission-list', kwargs={'username': self.user.username}) + "view-submission-list", kwargs={"username": self.user.username} + ) self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) self._download_submission_url = reverse( - 'view-download-submission', - kwargs={'username': self.user.username}) + "view-download-submission", kwargs={"username": self.user.username} + ) self._form_upload_url = reverse( - 'form-upload', kwargs={'username': self.user.username}) + "form-upload", kwargs={"username": self.user.username} + ) def test_view_submission_list(self): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} + params = {"formId": self.xform.id_string} request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -56,27 +58,30 @@ def test_view_submission_list(self): response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.this_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with open(submission_list_path, encoding='utf-8') as f: + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual(response.content.decode("utf-8"), expected_submission_list) def test_view_submission_list_w_deleted_submission(self): self._publish_xml_form() self._make_submissions() - uuid = 'f3d8dc65-91a6-4d0f-9e97-802128083390' - Instance.objects.filter(uuid=uuid).order_by('id').delete() - params = {'formId': self.xform.id_string} + uuid = "f3d8dc65-91a6-4d0f-9e97-802128083390" + Instance.objects.filter(uuid=uuid).order_by("id").delete() + params = {"formId": self.xform.id_string} request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -85,39 +90,43 @@ def test_view_submission_list_w_deleted_submission(self): response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submissionList-4.xml') + self.this_directory, + "fixtures", + "transportation", + "view", + "submissionList-4.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES - 1) last_index = instances[instances.count() - 1].pk - with open(submission_list_path, encoding='utf-8') as f: + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) - - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': uuid} - params = {'formId': formId} + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual(response.content.decode("utf-8"), expected_submission_list) + + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": uuid} + ) + params = {"formId": formId} response = self.client.get(self._download_submission_url, data=params) self.assertTrue(response.status_code, 404) def test_view_submission_list_OtherUser(self): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} + params = {"formId": self.xform.id_string} # deno cannot view bob's submissionList - self._create_user('deno', 'deno') + self._create_user("deno", "deno") request = self.factory.get(self._submission_list_url, params) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = view_submission_list(request, self.user.username) self.assertEqual(response.status_code, 403) @@ -137,8 +146,8 @@ def get_last_index(xform, last_index=None): self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} - params['numEntries'] = 2 + params = {"formId": self.xform.id_string} + params["numEntries"] = 2 instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) @@ -156,37 +165,42 @@ def get_last_index(xform, last_index=None): if index > 2: last_index = get_last_index(self.xform, last_index) - filename = 'submissionList-%s.xml' % index + filename = "submissionList-%s.xml" % index if index == 4: - self.assertEqual(response.content.decode('utf-8'), - last_expected_submission_list) + self.assertEqual( + response.content.decode("utf-8"), last_expected_submission_list + ) continue # set cursor for second request - params['cursor'] = last_index + params["cursor"] = last_index submission_list_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', filename) - with open(submission_list_path, encoding='utf-8') as f: + self.this_directory, "fixtures", "transportation", "view", filename + ) + with open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - last_expected_submission_list = expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) - self.assertEqual(response.content.decode('utf-8'), - expected_submission_list) + last_expected_submission_list = ( + expected_submission_list + ) = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) + self.assertEqual( + response.content.decode("utf-8"), expected_submission_list + ) last_index += 2 def test_view_downloadSubmission(self): self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} request = self.factory.get(self._download_submission_url, params) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 401) @@ -195,49 +209,55 @@ def test_view_downloadSubmission(self): response = view_download_submission(request, self.user.username) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with open(download_submission_path, encoding='utf-8') as f: + self.this_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_OtherUser(self): self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} # deno cannot view bob's downloadSubmission - self._create_user('deno', 'deno') + self._create_user("deno", "deno") request = self.factory.get(self._download_submission_url, params) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = view_download_submission(request, self.user.username) self.assertEqual(response.status_code, 403) def test_publish_xml_form_OtherUser(self): # deno cannot publish form to bob's account - self._create_user('deno', 'deno') + self._create_user("deno", "deno") count = XForm.objects.count() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) - auth = DigestAuth('deno', 'deno') + auth = DigestAuth("deno", "deno") request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) self.assertNotEqual(XForm.objects.count(), count + 1) @@ -245,40 +265,38 @@ def test_publish_xml_form_OtherUser(self): def test_publish_xml_form_where_filename_is_not_id_string(self): form_def_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'Transportation Form.xml') + self.this_directory, "fixtures", "transportation", "Transportation Form.xml" + ) count = XForm.objects.count() - with open(form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) - self.assertContains( - response, "successfully published.", status_code=201) + self.assertContains(response, "successfully published.", status_code=201) self.assertEqual(XForm.objects.count(), count + 1) def _publish_xml_form(self): count = XForm.objects.count() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) request.META.update(auth(request.META, response)) response = form_upload(request, username=self.user.username) - self.assertContains( - response, "successfully published.", status_code=201) + self.assertContains(response, "successfully published.", status_code=201) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.xform = XForm.objects.order_by("pk").reverse()[0] def test_form_upload(self): self._publish_xml_form() - with open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} request = self.factory.post(self._form_upload_url, params) response = form_upload(request, username=self.user.username) self.assertEqual(response.status_code, 401) @@ -287,25 +305,24 @@ def test_form_upload(self): response = form_upload(request, username=self.user.username) self.assertContains( response, - u'Form with this id or SMS-keyword already exists', - status_code=400) + "Form with this id or SMS-keyword already exists", + status_code=400, + ) def test_submission_with_instance_id_on_root_node(self): self._publish_xml_form() - message = u"Successful submission." - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - self.assertRaises( - Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) + message = "Successful submission." + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + self.assertRaises(Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) submission_path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'view', 'submission.xml') + self.this_directory, "fixtures", "transportation", "view", "submission.xml" + ) count = Instance.objects.count() - with open(submission_path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with open(submission_path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} self.factory = APIRequestFactory() request = self.factory.post(self._submission_url, post_data) - request.user = authenticate(username='bob', - password='bob') + request.user = authenticate(username="bob", password="bob") response = submission(request, username=self.user.username) self.assertContains(response, message, status_code=201) self.assertContains(response, instanceId, status_code=201) diff --git a/onadata/apps/logger/tests/test_digest_authentication.py b/onadata/apps/logger/tests/test_digest_authentication.py index 3d0b594461..421059c165 100644 --- a/onadata/apps/logger/tests/test_digest_authentication.py +++ b/onadata/apps/logger/tests/test_digest_authentication.py @@ -14,7 +14,7 @@ from onadata.apps.api.models.odk_token import ODKToken -ODK_TOKEN_STORAGE = 'onadata.apps.api.storage.ODKTokenAccountStorage' +ODK_TOKEN_STORAGE = "onadata.apps.api.storage.ODKTokenAccountStorage" class TestDigestAuthentication(TestBase): @@ -29,45 +29,49 @@ def test_authenticated_submissions(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() auth = DigestAuth(self.login_username, self.login_password) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) def _set_require_auth(self, auth=True): - profile, created = \ - UserProfile.objects.get_or_create(user=self.user) + profile, created = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = auth profile.save() def test_fail_authenticated_submissions_to_wrong_account(self): - username = 'dennis' + username = "dennis" # set require_auth b4 we switch user self._set_require_auth() self._create_user_and_login(username=username, password=username) self._set_require_auth() s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=DigestAuth('alice', 'alice')) + self._make_submission( + xml_submission_file_path, add_uuid=True, auth=DigestAuth("alice", "alice") + ) # Authentication required self.assertEqual(self.response.status_code, 401) - auth = DigestAuth('dennis', 'dennis') + auth = DigestAuth("dennis", "dennis") with self.assertRaises(Http404): - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) - @override_settings( - DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE - ) + @override_settings(DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE) def test_digest_authentication_with_odk_token_storage(self): """ Test that a valid Digest request with as the auth email:odk_token @@ -75,42 +79,45 @@ def test_digest_authentication_with_odk_token_storage(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() # Set email for user - self.user.email = 'bob@bob.test' + self.user.email = "bob@bob.test" self.user.save() odk_token = ODKToken.objects.create(user=self.user) # The value odk_token.key is hashed we need to have the raw_key # In order to authenticate with DigestAuth - fernet = Fernet(getattr(settings, 'ODK_TOKEN_FERNET_KEY')) - raw_key = fernet.decrypt(odk_token.key.encode('utf-8')).decode('utf-8') + fernet = Fernet(getattr(settings, "ODK_TOKEN_FERNET_KEY")) + raw_key = fernet.decrypt(odk_token.key.encode("utf-8")).decode("utf-8") auth = DigestAuth(self.user.email, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) # Test can authenticate with username:token s = self.surveys[1] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) auth = DigestAuth(self.user.username, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 201) - @override_settings( - ODK_KEY_LIFETIME=1, - DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE - ) + @override_settings(ODK_KEY_LIFETIME=1, DIGEST_ACCOUNT_BACKEND=ODK_TOKEN_STORAGE) def test_fails_authentication_past_odk_token_expiry(self): """ Test that a Digest authenticated request using an ODK Token that has @@ -118,13 +125,17 @@ def test_fails_authentication_past_odk_token_expiry(self): """ s = self.surveys[0] xml_submission_file_path = os.path.join( - self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml' + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", ) self._set_require_auth() # Set email for user - self.user.email = 'bob@bob.test' + self.user.email = "bob@bob.test" self.user.save() odk_token = ODKToken.objects.create(user=self.user) @@ -134,10 +145,9 @@ def test_fails_authentication_past_odk_token_expiry(self): # The value odk_token.key is hashed we need to have the raw_key # In order to authenticate with DigestAuth - fernet = Fernet(getattr(settings, 'ODK_TOKEN_FERNET_KEY')) - raw_key = fernet.decrypt(odk_token.key.encode('utf-8')).decode('utf-8') + fernet = Fernet(getattr(settings, "ODK_TOKEN_FERNET_KEY")) + raw_key = fernet.decrypt(odk_token.key.encode("utf-8")).decode("utf-8") auth = DigestAuth(self.user.email, raw_key) - self._make_submission(xml_submission_file_path, add_uuid=True, - auth=auth) + self._make_submission(xml_submission_file_path, add_uuid=True, auth=auth) self.assertEqual(self.response.status_code, 401) diff --git a/onadata/apps/logger/tests/test_encrypted_submissions.py b/onadata/apps/logger/tests/test_encrypted_submissions.py index ce85c0a86a..80d9f7809a 100644 --- a/onadata/apps/logger/tests/test_encrypted_submissions.py +++ b/onadata/apps/logger/tests/test_encrypted_submissions.py @@ -25,38 +25,47 @@ def setUp(self): super(TestEncryptedForms, self).setUp() self._create_user_and_login() self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) def test_encrypted_submissions(self): """ Test encrypted submissions. """ - self._publish_xls_file(os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'transportation_encrypted.xlsx' - )) - xform = XForm.objects.get(id_string='transportation_encrypted') + self._publish_xls_file( + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_encrypted.xlsx", + ) + ) + xform = XForm.objects.get(id_string="transportation_encrypted") self.assertTrue(xform.encrypted) uuid = "c15252fe-b6f3-4853-8f04-bf89dc73985a" with self.assertRaises(Instance.DoesNotExist): Instance.objects.get(uuid=uuid) - message = u"Successful submission." + message = "Successful submission." files = {} - for filename in ['submission.xml', 'submission.xml.enc']: + for filename in ["submission.xml", "submission.xml.enc"]: files[filename] = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances_encrypted', filename) + self.this_directory, + "fixtures", + "transportation", + "instances_encrypted", + filename, + ) count = Instance.objects.count() acount = Attachment.objects.count() - with open(files['submission.xml.enc'], 'rb') as encryped_file: - with open(files['submission.xml'], 'rb') as f: + with open(files["submission.xml.enc"], "rb") as encryped_file: + with open(files["submission.xml"], "rb") as f: post_data = { - 'xml_submission_file': f, - 'submission.xml.enc': encryped_file} + "xml_submission_file": f, + "submission.xml.enc": encryped_file, + } self.factory = APIRequestFactory() request = self.factory.post(self._submission_url, post_data) - request.user = authenticate(username='bob', - password='bob') + request.user = authenticate(username="bob", password="bob") response = submission(request, username=self.user.username) self.assertContains(response, message, status_code=201) self.assertEqual(Instance.objects.count(), count + 1) @@ -75,7 +84,7 @@ def test_encrypted_multiple_files(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx" + "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -84,36 +93,36 @@ def test_encrypted_multiple_files(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml" + "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml", ) encrypted_xml_submission = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/submission.xml.enc" + "../fixtures/tutorial_encrypted/instances/submission.xml.enc", + ) + self._make_submission_w_attachment( + xml_submission_file_path, encrypted_xml_submission + ) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=201 ) - self._make_submission_w_attachment(xml_submission_file_path, - encrypted_xml_submission) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=201) # load xml file to parse and compare # expected_list = [{u'file': u'1483528430996.jpg.enc'}, # {u'file': u'1483528445767.jpg.enc'}] - instance = Instance.objects.filter().order_by('id').last() + instance = Instance.objects.filter().order_by("id").last() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 1) self.assertFalse(instance.media_all_received) media_file_1 = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/1483528430996.jpg.enc" + "../fixtures/tutorial_encrypted/instances/1483528430996.jpg.enc", + ) + self._make_submission_w_attachment(xml_submission_file_path, media_file_1) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=202 ) - self._make_submission_w_attachment(xml_submission_file_path, - media_file_1) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=202) instance.refresh_from_db() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 2) @@ -121,13 +130,12 @@ def test_encrypted_multiple_files(self): media_file_2 = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/1483528445767.jpg.enc" + "../fixtures/tutorial_encrypted/instances/1483528445767.jpg.enc", + ) + self._make_submission_w_attachment(xml_submission_file_path, media_file_2) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=202 ) - self._make_submission_w_attachment(xml_submission_file_path, - media_file_2) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=202) instance.refresh_from_db() self.assertEqual(instance.total_media, 3) self.assertEqual(instance.media_count, 3) diff --git a/onadata/apps/logger/tests/test_form_list.py b/onadata/apps/logger/tests/test_form_list.py index a2a01cd19f..a02aa03c8e 100644 --- a/onadata/apps/logger/tests/test_form_list.py +++ b/onadata/apps/logger/tests/test_form_list.py @@ -13,8 +13,8 @@ def setUp(self): def test_returns_200_for_owner(self): self._set_require_auth() - request = self.factory.get('/') - auth = DigestAuth('bob', 'bob') + request = self.factory.get("/") + auth = DigestAuth("bob", "bob") response = formList(request, self.user.username) request.META.update(auth(request.META, response)) response = formList(request, self.user.username) @@ -22,23 +22,26 @@ def test_returns_200_for_owner(self): def test_return_401_for_anon_when_require_auth_true(self): self._set_require_auth() - request = self.factory.get('/') + request = self.factory.get("/") response = formList(request, self.user.username) self.assertEqual(response.status_code, 401) def test_returns_200_for_authenticated_non_owner(self): self._set_require_auth() - credentials = ('alice', 'alice',) + credentials = ( + "alice", + "alice", + ) self._create_user(*credentials) - auth = DigestAuth('alice', 'alice') - request = self.factory.get('/') + auth = DigestAuth("alice", "alice") + request = self.factory.get("/") response = formList(request, self.user.username) request.META.update(auth(request.META, response)) response = formList(request, self.user.username) self.assertEqual(response.status_code, 200) def test_show_for_anon_when_require_auth_false(self): - request = self.factory.get('/') + request = self.factory.get("/") request.user = AnonymousUser() response = formList(request, self.user.username) self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/tests/test_importing_database.py b/onadata/apps/logger/tests/test_importing_database.py index cdb185ef9f..bc671560d9 100644 --- a/onadata/apps/logger/tests/test_importing_database.py +++ b/onadata/apps/logger/tests/test_importing_database.py @@ -11,31 +11,39 @@ CUR_PATH = os.path.abspath(__file__) CUR_DIR = os.path.dirname(CUR_PATH) -DB_FIXTURES_PATH = os.path.join(CUR_DIR, 'data_from_sdcard') +DB_FIXTURES_PATH = os.path.join(CUR_DIR, "data_from_sdcard") def images_count(username="bob"): images = glob.glob( - os.path.join(settings.MEDIA_ROOT, username, 'attachments', '*', '*')) + os.path.join(settings.MEDIA_ROOT, username, "attachments", "*", "*") + ) return len(images) class TestImportingDatabase(TestBase): - def setUp(self): TestBase.setUp(self) self._publish_xls_file( os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "test_forms", "tutorial.xlsx")) + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "test_forms", + "tutorial.xlsx", + ) + ) def tearDown(self): # delete everything we imported Instance.objects.all().delete() # ? if settings.TESTING_MODE: images = glob.glob( - os.path.join(settings.MEDIA_ROOT, self.user.username, - 'attachments', '*', '*')) + os.path.join( + settings.MEDIA_ROOT, self.user.username, "attachments", "*", "*" + ) + ) for image in images: os.remove(image) @@ -56,8 +64,9 @@ def test_importing_b1_and_b2(self): initial_instance_count = Instance.objects.count() initial_image_count = images_count() - import_instances_from_zip(os.path.join( - DB_FIXTURES_PATH, "bulk_submission.zip"), self.user) + import_instances_from_zip( + os.path.join(DB_FIXTURES_PATH, "bulk_submission.zip"), self.user + ) instance_count = Instance.objects.count() image_count = images_count() @@ -71,31 +80,29 @@ def test_importing_b1_and_b2(self): def test_badzipfile_import(self): total, success, errors = import_instances_from_zip( - os.path.join( - CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user) + os.path.join(CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user + ) self.assertEqual(total, 0) self.assertEqual(success, 0) - expected_errors = [u'File is not a zip file'] + expected_errors = ["File is not a zip file"] self.assertEqual(errors, expected_errors) def test_bulk_import_post(self): zip_file_path = os.path.join( - DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip") - url = reverse(bulksubmission, kwargs={ - "username": self.user.username - }) + DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip" + ) + url = reverse(bulksubmission, kwargs={"username": self.user.username}) with open(zip_file_path, "rb") as zip_file: - post_data = {'zip_submission_file': zip_file} + post_data = {"zip_submission_file": zip_file} response = self.client.post(url, post_data) self.assertEqual(response.status_code, 200) def test_bulk_import_post_with_username_in_uppercase(self): zip_file_path = os.path.join( - DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip") - url = reverse(bulksubmission, kwargs={ - "username": self.user.username.upper() - }) + DB_FIXTURES_PATH, "bulk_submission_w_extra_instance.zip" + ) + url = reverse(bulksubmission, kwargs={"username": self.user.username.upper()}) with open(zip_file_path, "rb") as zip_file: - post_data = {'zip_submission_file': zip_file} + post_data = {"zip_submission_file": zip_file} response = self.client.post(url, post_data) self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/logger/tests/test_instance_creation.py b/onadata/apps/logger/tests/test_instance_creation.py index 574cf814cc..9d8535672e 100644 --- a/onadata/apps/logger/tests/test_instance_creation.py +++ b/onadata/apps/logger/tests/test_instance_creation.py @@ -37,12 +37,10 @@ def create_post_data(path): def get_absolute_path(subdirectory): - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), subdirectory) + return os.path.join(os.path.dirname(os.path.abspath(__file__)), subdirectory) class TestInstanceCreation(TestCase): - def setUp(self): self.user = User.objects.create(username="bob") profile, c = UserProfile.objects.get_or_create(user=self.user) @@ -52,34 +50,46 @@ def setUp(self): absolute_path = get_absolute_path("forms") open_forms = open_all_files(absolute_path) - self.json = '{"default_language": "default", ' \ - '"id_string": "Water_2011_03_17", "children": [], ' \ - '"name": "Water_2011_03_17", ' \ - '"title": "Water_2011_03_17", "type": "survey"}' + self.json = ( + '{"default_language": "default", ' + '"id_string": "Water_2011_03_17", "children": [], ' + '"name": "Water_2011_03_17", ' + '"title": "Water_2011_03_17", "type": "survey"}' + ) for path, open_file in open_forms.items(): XForm.objects.create( - xml=open_file.read(), user=self.user, json=self.json, - require_auth=False, project=self.project) + xml=open_file.read(), + user=self.user, + json=self.json, + require_auth=False, + project=self.project, + ) open_file.close() self._create_water_translated_form() def _create_water_translated_form(self): - f = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Water_Translated_2011_03_10.xml" - )) + f = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Water_Translated_2011_03_10.xml", + ) + ) xml = f.read() f.close() self.xform = XForm.objects.create( - xml=xml, user=self.user, json=self.json, project=self.project) + xml=xml, user=self.user, json=self.json, project=self.project + ) def test_form_submission(self): # no more submission to non-existent form, # setUp ensures the Water_Translated_2011_03_10 xform is valid - f = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Water_Translated_2011_03_10_2011-03-10_14-38-28.xml")) + f = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Water_Translated_2011_03_10_2011-03-10_14-38-28.xml", + ) + ) xml = f.read() f.close() Instance.objects.create(xml=xml, user=self.user, xform=self.xform) @@ -89,14 +99,16 @@ def test_data_submission(self): for subdirectory in subdirectories: path = get_absolute_path(subdirectory) postdata = create_post_data(path) - response = self.client.post('/bob/submission', postdata) + response = self.client.post("/bob/submission", postdata) self.failUnlessEqual(response.status_code, 201) def test_submission_for_missing_form(self): - xml_file = open(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "Health_2011_03_13_invalid_id_string.xml" - )) + xml_file = open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "Health_2011_03_13_invalid_id_string.xml", + ) + ) postdata = {"xml_submission_file": xml_file} - response = self.client.post('/bob/submission', postdata) + response = self.client.post("/bob/submission", postdata) self.failUnlessEqual(response.status_code, 404) diff --git a/onadata/apps/logger/tests/test_parsing.py b/onadata/apps/logger/tests/test_parsing.py index 7cc1d60639..48075fb213 100644 --- a/onadata/apps/logger/tests/test_parsing.py +++ b/onadata/apps/logger/tests/test_parsing.py @@ -5,19 +5,26 @@ from xml.dom import minidom from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.xform_instance_parser import XFormInstanceParser,\ - xpath_from_xml_node -from onadata.apps.logger.xform_instance_parser import get_uuid_from_xml,\ - get_meta_from_xml, get_deprecated_uuid_from_xml +from onadata.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + xpath_from_xml_node, +) +from onadata.apps.logger.xform_instance_parser import ( + get_uuid_from_xml, + get_meta_from_xml, + get_deprecated_uuid_from_xml, +) from onadata.libs.utils.common_tags import XFORM_ID_STRING from onadata.apps.logger.models.xform import XForm -from onadata.apps.logger.xform_instance_parser import _xml_node_to_dict,\ - clean_and_parse_xml +from onadata.apps.logger.xform_instance_parser import ( + _xml_node_to_dict, + clean_and_parse_xml, +) -XML = u"xml" -DICT = u"dict" -FLAT_DICT = u"flat_dict" +XML = "xml" +DICT = "dict" +FLAT_DICT = "flat_dict" ID = XFORM_ID_STRING @@ -27,7 +34,7 @@ def _publish_and_submit_new_repeats(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/new_repeats.xlsx" + "../fixtures/new_repeats/new_repeats.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -36,8 +43,7 @@ def _publish_and_submit_new_repeats(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/instances/" - "new_repeats_2012-07-05-14-33-53.xml" + "../fixtures/new_repeats/instances/" "new_repeats_2012-07-05-14-33-53.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) @@ -52,85 +58,88 @@ def test_parse_xform_nested_repeats(self): parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() expected_dict = { - u'new_repeats': { - u'info': - { - u'age': u'80', - u'name': u'Adam' - }, - u'kids': - { - u'kids_details': - [ - { - u'kids_age': u'50', - u'kids_name': u'Abel' - }, + "new_repeats": { + "info": {"age": "80", "name": "Adam"}, + "kids": { + "kids_details": [ + {"kids_age": "50", "kids_name": "Abel"}, ], - u'has_kids': u'1' + "has_kids": "1", }, - u'web_browsers': u'chrome ie', - u'gps': u'-1.2627557 36.7926442 0.0 30.0' + "web_browsers": "chrome ie", + "gps": "-1.2627557 36.7926442 0.0 30.0", } } self.assertEqual(dict, expected_dict) flat_dict = parser.to_flat_dict() expected_flat_dict = { - u'gps': u'-1.2627557 36.7926442 0.0 30.0', - u'kids/kids_details': - [ + "gps": "-1.2627557 36.7926442 0.0 30.0", + "kids/kids_details": [ { - u'kids/kids_details/kids_name': u'Abel', - u'kids/kids_details/kids_age': u'50' + "kids/kids_details/kids_name": "Abel", + "kids/kids_details/kids_age": "50", } ], - u'kids/has_kids': u'1', - u'info/age': u'80', - u'web_browsers': u'chrome ie', - u'info/name': u'Adam' + "kids/has_kids": "1", + "info/age": "80", + "web_browsers": "chrome ie", + "info/name": "Adam", } self.assertEqual(flat_dict, expected_flat_dict) def test_xpath_from_xml_node(self): - xml_str = '' \ - 'c911d71ce1ac48478e5f8bac99addc4e' \ - '-1.2625149 36.7924478 0.0 30.0' \ - 'Yo' \ - '-1.2625072 36.7924328 0.0 30.0' \ - 'What' + xml_str = ( + "' + "c911d71ce1ac48478e5f8bac99addc4e" + "-1.2625149 36.7924478 0.0 30.0" + "Yo" + "-1.2625072 36.7924328 0.0 30.0" + "What" + ) clean_xml_str = xml_str.strip() - clean_xml_str = re.sub(r">\s+<", u"><", clean_xml_str) + clean_xml_str = re.sub(r">\s+<", "><", clean_xml_str) root_node = minidom.parseString(clean_xml_str).documentElement # get the first top-level gps element gps_node = root_node.firstChild.nextSibling - self.assertEqual(gps_node.nodeName, u'gps') + self.assertEqual(gps_node.nodeName, "gps") # get the info element within the gps element - info_node = gps_node.getElementsByTagName(u'info')[0] + info_node = gps_node.getElementsByTagName("info")[0] # create an xpath that should look like gps/info xpath = xpath_from_xml_node(info_node) - self.assertEqual(xpath, u'gps/info') + self.assertEqual(xpath, "gps/info") def test_get_meta_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_meta_from_xml(xml_str, "instanceID") - self.assertEqual(instanceID, - "uuid:2d8c59eb-94e9-485d-a679-b28ffe2e9b98") + self.assertEqual(instanceID, "uuid:2d8c59eb-94e9-485d-a679-b28ffe2e9b98") deprecatedID = get_meta_from_xml(xml_str, "deprecatedID") self.assertEqual(deprecatedID, "uuid:729f173c688e482486a48661700455ff") def test_get_meta_from_xml_without_uuid_returns_none(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_meta_from_xml(xml_str, "instanceID") self.assertIsNone(instanceID) @@ -138,9 +147,15 @@ def test_get_meta_from_xml_without_uuid_returns_none(self): def test_get_uuid_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() instanceID = get_uuid_from_xml(xml_str) self.assertEqual(instanceID, "729f173c688e482486a48661700455ff") @@ -148,9 +163,15 @@ def test_get_uuid_from_xml(self): def test_get_deprecated_uuid_from_xml(self): with open( os.path.join( - os.path.dirname(__file__), "..", "fixtures", "tutorial", - "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"), - "r") as xml_file: + os.path.dirname(__file__), + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", + ), + "r", + ) as xml_file: xml_str = xml_file.read() deprecatedID = get_deprecated_uuid_from_xml(xml_str) self.assertEqual(deprecatedID, "729f173c688e482486a48661700455ff") @@ -160,7 +181,7 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/new_repeats.xlsx" + "../fixtures/new_repeats/new_repeats.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -169,8 +190,7 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/new_repeats/instances/" - "multiple_nodes_error.xml" + "../fixtures/new_repeats/instances/" "multiple_nodes_error.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(201, self.response.status_code) @@ -180,7 +200,7 @@ def test_multiple_media_files_on_encrypted_form(self): # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx" + "../fixtures/tutorial_encrypted/tutorial_encrypted.xlsx", ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) @@ -189,12 +209,12 @@ def test_multiple_media_files_on_encrypted_form(self): # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml" + "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml", ) self._make_submission(xml_submission_file_path) - self.assertNotContains(self.response, - "Multiple nodes with the same name", - status_code=201) + self.assertNotContains( + self.response, "Multiple nodes with the same name", status_code=201 + ) # load xml file to parse and compare xml_file = open(xml_submission_file_path) @@ -204,28 +224,30 @@ def test_multiple_media_files_on_encrypted_form(self): parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() - expected_list = [{u'file': u'1483528430996.jpg.enc'}, - {u'file': u'1483528445767.jpg.enc'}] - self.assertEqual(dict.get('data').get('media'), expected_list) + expected_list = [ + {"file": "1483528430996.jpg.enc"}, + {"file": "1483528445767.jpg.enc"}, + ] + self.assertEqual(dict.get("data").get("media"), expected_list) flat_dict = parser.to_flat_dict() - expected_flat_list = [{u'media/file': u'1483528430996.jpg.enc'}, - {u'media/file': u'1483528445767.jpg.enc'}] - self.assertEqual(flat_dict.get('media'), expected_flat_list) + expected_flat_list = [ + {"media/file": "1483528430996.jpg.enc"}, + {"media/file": "1483528445767.jpg.enc"}, + ] + self.assertEqual(flat_dict.get("media"), expected_flat_list) def test_xml_repeated_nodes_to_dict(self): xml_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../fixtures/repeated_nodes.xml" + os.path.dirname(os.path.abspath(__file__)), "../fixtures/repeated_nodes.xml" ) json_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../fixtures/repeated_nodes_expected_results.json" + "../fixtures/repeated_nodes_expected_results.json", ) with open(xml_file) as file: xml_dict = _xml_node_to_dict(clean_and_parse_xml(file.read())) - self.assertTrue(xml_dict['#document']['RW_OUNIS_2016']['S2A']) - self.assertEqual(3, len( - xml_dict['#document']['RW_OUNIS_2016']['S2A'])) + self.assertTrue(xml_dict["#document"]["RW_OUNIS_2016"]["S2A"]) + self.assertEqual(3, len(xml_dict["#document"]["RW_OUNIS_2016"]["S2A"])) with open(json_file) as file: self.assertEqual(json.loads(file.read()), xml_dict) diff --git a/onadata/apps/logger/tests/test_publish_xls.py b/onadata/apps/logger/tests/test_publish_xls.py index 36ee8a39a1..10192e3087 100644 --- a/onadata/apps/logger/tests/test_publish_xls.py +++ b/onadata/apps/logger/tests/test_publish_xls.py @@ -11,13 +11,12 @@ class TestPublishXLS(TestBase): - def test_publish_xls(self): xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) form = XForm.objects.get() self.assertFalse(form.require_auth) @@ -25,26 +24,36 @@ def test_publish_xls(self): def test_publish_xls_replacement(self): count = XForm.objects.count() xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") - call_command('publish_xls', xls_file_path, self.user.username) + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) count = XForm.objects.count() xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation_updated.xlsx") + self.this_directory, + "fixtures", + "transportation", + "transportation_updated.xlsx", + ) # call command without replace param with self.assertRaises(CommandError): - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) # now we call the command with the replace param - call_command( - 'publish_xls', xls_file_path, self.user.username, replace=True) + call_command("publish_xls", xls_file_path, self.user.username, replace=True) # count should remain the same self.assertEqual(XForm.objects.count(), count) # check if the extra field has been added - self.xform = XForm.objects.order_by('id').reverse()[0] - is_updated_form = len([e.name for e in self.xform.survey_elements - if e.name == u'preferred_means']) > 0 + self.xform = XForm.objects.order_by("id").reverse()[0] + is_updated_form = ( + len( + [ + e.name + for e in self.xform.survey_elements + if e.name == "preferred_means" + ] + ) + > 0 + ) self.assertTrue(is_updated_form) def test_xform_hash(self): @@ -59,12 +68,12 @@ def test_xform_hash(self): self.assertFalse(self.xform.hash == "" or self.xform.hash is None) self.assertEqual(self.xform.hash, self.xform.get_hash()) # test that the md5 value of the hash is as expected - calculated_hash = md5(self.xform.xml.encode('utf8')).hexdigest() + calculated_hash = md5(self.xform.xml.encode("utf8")).hexdigest() self.assertEqual(self.xform.hash[4:], calculated_hash) # assert that the hash changes when you change the form title xform_old_hash = self.xform.hash self.xform.title = "Hunter 2 Rules" - self.xform.save(update_fields=['title']) + self.xform.save(update_fields=["title"]) self.assertFalse(self.xform.hash == "" or self.xform.hash is None) self.assertFalse(self.xform.hash == xform_old_hash) @@ -75,8 +84,9 @@ def test_report_exception_with_exc_info(self): except Exception as e: exc_info = sys.exc_info() try: - report_exception(subject="Test report exception", info=e, - exc_info=exc_info) + report_exception( + subject="Test report exception", info=e, exc_info=exc_info + ) except Exception as e: raise AssertionError("%s" % e) @@ -89,10 +99,10 @@ def test_report_exception_without_exc_info(self): def test_publish_xls_version(self): xls_file_path = os.path.join( - self.this_directory, "fixtures", - "transportation", "transportation.xlsx") + self.this_directory, "fixtures", "transportation", "transportation.xlsx" + ) count = XForm.objects.count() - call_command('publish_xls', xls_file_path, self.user.username) + call_command("publish_xls", xls_file_path, self.user.username) self.assertEqual(XForm.objects.count(), count + 1) form = XForm.objects.get() self.assertIsNotNone(form.version) diff --git a/onadata/apps/logger/tests/test_simple_submission.py b/onadata/apps/logger/tests/test_simple_submission.py index b8ab174981..c8fd456872 100644 --- a/onadata/apps/logger/tests/test_simple_submission.py +++ b/onadata/apps/logger/tests/test_simple_submission.py @@ -7,9 +7,7 @@ from onadata.apps.logger.xform_instance_parser import DuplicateInstance from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.libs.utils.logger_tools import ( - create_instance, safe_create_instance -) +from onadata.libs.utils.logger_tools import create_instance, safe_create_instance from onadata.libs.utils.user_auth import get_user_default_project @@ -18,8 +16,9 @@ class TempFileProxy(object): create_instance will be looking for a file object, with "read" and "close" methods. """ + def __init__(self, content): - self.content = content.encode('utf-8') + self.content = content.encode("utf-8") def read(self): return self.content @@ -37,42 +36,51 @@ def _get_xml_for_form(self, xform): xform.save() def _submit_at_hour(self, hour): - st_xml = '2012-01-11T%d:00:00.000+00'\ - '' % hour + st_xml = ( + "2012-01-11T%d:00:00.000+00' + "" % hour + ) try: create_instance(self.user.username, TempFileProxy(st_xml), []) except DuplicateInstance: pass def _submit_simple_yes(self): - create_instance(self.user.username, TempFileProxy( - '' - 'Yes'), []) + create_instance( + self.user.username, + TempFileProxy( + "" + "Yes" + ), + [], + ) def setUp(self): - self.user = User.objects.create( - username="admin", email="sample@example.com") + self.user = User.objects.create(username="admin", email="sample@example.com") self.project = UserProfile.objects.create(user=self.user) self.user.set_password("pass") self.project = get_user_default_project(self.user) self.xform1 = DataDictionary() self.xform1.user = self.user self.xform1.project = self.project - self.xform1.json = '{"id_string": "yes_or_no_form", "children": '\ - '[{"name": '\ - '"yesno", "label": "Yes or no?", "type": "text"}],'\ - ' "name": "yes_or_no", "title": "yes_or_no", "type'\ - '": "survey"}'.strip() + self.xform1.json = ( + '{"id_string": "yes_or_no_form", "children": ' + '[{"name": ' + '"yesno", "label": "Yes or no?", "type": "text"}],' + ' "name": "yes_or_no", "title": "yes_or_no", "type' + '": "survey"}'.strip() + ) self.xform2 = DataDictionary() self.xform2.user = self.user self.xform2.project = self.project - self.xform2.json = '{"id_string": "start_time_form", "children": '\ - '[{"name":'\ - '"start_time", "type": "start"}], "name": "start_t'\ - 'ime_form", "title": "start_time_form",'\ - '"type": "survey"}'\ - .strip() + self.xform2.json = ( + '{"id_string": "start_time_form", "children": ' + '[{"name":' + '"start_time", "type": "start"}], "name": "start_t' + 'ime_form", "title": "start_time_form",' + '"type": "survey"}'.strip() + ) self._get_xml_for_form(self.xform1) self._get_xml_for_form(self.xform2) @@ -109,13 +117,13 @@ def test_start_time_submissions(self): self.assertEqual(2, self.xform2.instances.count()) def test_corrupted_submission(self): - """Test xml submissions that contain unicode characters. - """ - xml = 'v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9\'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^\'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13\'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh\'|?\xff\xff\xff\xff\xd5\'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=' # noqa + """Test xml submissions that contain unicode characters.""" + xml = "v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h\"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{\"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh'|?\xff\xff\xff\xff\xd5'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=" # noqa - request = RequestFactory().post('/') + request = RequestFactory().post("/") request.user = self.user error, instance = safe_create_instance( - self.user.username, TempFileProxy(xml), None, None, request) - text = 'Improperly formatted XML.' + self.user.username, TempFileProxy(xml), None, None, request + ) + text = "Improperly formatted XML." self.assertContains(error, text, status_code=400) diff --git a/onadata/apps/logger/tests/test_update_xform_uuid.py b/onadata/apps/logger/tests/test_update_xform_uuid.py index 10ed529eb6..3752708f64 100644 --- a/onadata/apps/logger/tests/test_update_xform_uuid.py +++ b/onadata/apps/logger/tests/test_update_xform_uuid.py @@ -14,7 +14,8 @@ def setUp(self): # self.csv_filepath = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "fixtures", "test_update_xform_uuids.csv" + "fixtures", + "test_update_xform_uuids.csv", ) # get the last defined uuid with open(self.csv_filepath, "r") as f: @@ -33,8 +34,7 @@ def test_fail_update_on_duplicate_uuid(self): self.xform.uuid = self.new_uuid self.xform.save() try: - update_xform_uuid(self.user.username, self.xform.id_string, - self.new_uuid) + update_xform_uuid(self.user.username, self.xform.id_string, self.new_uuid) except DuplicateUUIDError: self.assertTrue(True) else: diff --git a/onadata/apps/logger/tests/test_webforms.py b/onadata/apps/logger/tests/test_webforms.py index 036a5d5fa7..4185abe0b2 100644 --- a/onadata/apps/logger/tests/test_webforms.py +++ b/onadata/apps/logger/tests/test_webforms.py @@ -11,7 +11,7 @@ from onadata.libs.utils.logger_tools import inject_instanceid -@urlmatch(netloc=r'(.*\.)?enketo\.ona\.io$') +@urlmatch(netloc=r"(.*\.)?enketo\.ona\.io$") def enketo_edit_mock(url, request): response = requests.Response() response.status_code = 201 @@ -25,30 +25,36 @@ def setUp(self): self._publish_transportation_form_and_submit_instance() def __load_fixture(self, *path): - with open(os.path.join(os.path.dirname(__file__), *path), 'r') as f: + with open(os.path.join(os.path.dirname(__file__), *path), "r") as f: return f.read() def test_edit_url(self): - instance = Instance.objects.order_by('id').reverse()[0] - edit_url = reverse(edit_data, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': instance.id - }) + instance = Instance.objects.order_by("id").reverse()[0] + edit_url = reverse( + edit_data, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "data_id": instance.id, + }, + ) with HTTMock(enketo_edit_mock): response = self.client.get(edit_url) self.assertEqual(response.status_code, 302) - self.assertEqual(response['location'], - 'https://hmh2a.enketo.ona.io') + self.assertEqual(response["location"], "https://hmh2a.enketo.ona.io") def test_inject_instanceid(self): """ Test that 1 and only 1 instance id exists or is injected """ instance = Instance.objects.all().reverse()[0] - xml_str = self.__load_fixture("..", "fixtures", "tutorial", - "instances", - "tutorial_2012-06-27_11-27-53.xml") + xml_str = self.__load_fixture( + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ) # test that we dont have an instance id uuid = get_uuid_from_xml(xml_str) self.assertIsNone(uuid) @@ -59,21 +65,19 @@ def test_inject_instanceid(self): def test_dont_inject_instanceid_if_exists(self): xls_file_path = os.path.join( - os.path.dirname(__file__), - '..', - 'fixtures', - 'tutorial', - 'tutorial.xlsx') + os.path.dirname(__file__), "..", "fixtures", "tutorial", "tutorial.xlsx" + ) self._publish_xls_file_and_set_xform(xls_file_path) xml_file_path = os.path.join( os.path.dirname(__file__), - '..', - 'fixtures', - 'tutorial', - 'instances', - 'tutorial_2012-06-27_11-27-53_w_uuid.xml') + "..", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53_w_uuid.xml", + ) self._make_submission(xml_file_path) - instance = Instance.objects.order_by('id').reverse()[0] + instance = Instance.objects.order_by("id").reverse()[0] injected_xml_str = inject_instanceid(instance.xml, instance.uuid) # check that the xml is unmodified self.assertEqual(instance.xml, injected_xml_str) diff --git a/onadata/apps/logger/xform_fs.py b/onadata/apps/logger/xform_fs.py index e2173db272..875f01e712 100644 --- a/onadata/apps/logger/xform_fs.py +++ b/onadata/apps/logger/xform_fs.py @@ -11,7 +11,7 @@ def __init__(self, filepath): @property def photos(self): - if not hasattr(self, '_photos'): + if not hasattr(self, "_photos"): available_photos = glob.glob(os.path.join(self.directory, "*.jpg")) self._photos = [] for photo_path in available_photos: @@ -22,7 +22,7 @@ def photos(self): @property def osm(self): - if not hasattr(self, '_osm'): + if not hasattr(self, "_osm"): available_osm = glob.glob(os.path.join(self.directory, "*.osm")) self._osm = [] for osm_path in available_osm: @@ -33,13 +33,10 @@ def osm(self): @property def metadata_directory(self): - if not hasattr(self, '_metadata_directory'): - instances_dir = os.path.join( - self.directory, "..", "..", "instances") - metadata_directory = os.path.join( - self.directory, "..", "..", "metadata") - if os.path.exists(instances_dir) and os.path.exists( - metadata_directory): + if not hasattr(self, "_metadata_directory"): + instances_dir = os.path.join(self.directory, "..", "..", "instances") + metadata_directory = os.path.join(self.directory, "..", "..", "metadata") + if os.path.exists(instances_dir) and os.path.exists(metadata_directory): self._metadata_directory = os.path.abspath(metadata_directory) else: self._metadata_directory = False @@ -47,8 +44,8 @@ def metadata_directory(self): @property def xml(self): - if not hasattr(self, '_xml'): - with open(self.path, 'r') as f: + if not hasattr(self, "_xml"): + with open(self.path, "r") as f: self._xml = f.read() return self._xml @@ -56,9 +53,9 @@ def xml(self): def is_valid_instance(cls, filepath): if not filepath.endswith(".xml"): return False - with open(filepath, 'r') as ff: + with open(filepath, "r") as ff: fxml = ff.read() - if not fxml.strip().startswith(' Date: Thu, 12 May 2022 04:10:04 +0300 Subject: [PATCH 205/234] logger: cleanup --- onadata/apps/logger/admin.py | 44 ++++++---- onadata/apps/logger/import_tools.py | 89 ++++++++++++-------- onadata/apps/logger/views.py | 13 +-- onadata/apps/logger/xform_fs.py | 38 ++++++--- onadata/apps/logger/xform_instance_parser.py | 2 +- 5 files changed, 114 insertions(+), 72 deletions(-) diff --git a/onadata/apps/logger/admin.py b/onadata/apps/logger/admin.py index a62cbc869f..d1a14f8cfd 100644 --- a/onadata/apps/logger/admin.py +++ b/onadata/apps/logger/admin.py @@ -1,38 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Logger admin module. +""" +from django.contrib import admin + from reversion.admin import VersionAdmin -from django.contrib import admin +from onadata.apps.logger.models import Project, XForm + + +class FilterByUserMixin: + """Filter queryset by ``request.user``.""" + + # A user should only see forms/projects that belong to him. + def get_queryset(self, request): + """Returns queryset filtered by the `request.user`.""" + queryset = super().get_queryset(request) + if request.user.is_superuser: + return queryset + return queryset.filter(**{self.user_lookup_field: request.user}) -from onadata.apps.logger.models import XForm, Project +class XFormAdmin(FilterByUserMixin, VersionAdmin, admin.ModelAdmin): + """Customise the XForm admin view.""" -class XFormAdmin(VersionAdmin, admin.ModelAdmin): exclude = ("user",) list_display = ("id_string", "downloadable", "shared") search_fields = ("id_string", "title") - - # A user should only see forms that belong to him. - def get_queryset(self, request): - qs = super(XFormAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + user_lookup_field = "user" admin.site.register(XForm, XFormAdmin) -class ProjectAdmin(VersionAdmin, admin.ModelAdmin): +class ProjectAdmin(FilterByUserMixin, VersionAdmin, admin.ModelAdmin): + """Customise the Project admin view.""" + list_max_show_all = 2000 list_select_related = ("organization",) ordering = ["name"] search_fields = ("name", "organization__username", "organization__email") - - # A user should only see projects that belong to him. - def get_queryset(self, request): - qs = super(ProjectAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - return qs.filter(organization=request.user) + user_lookup_field = "organization" admin.site.register(Project, ProjectAdmin) diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index ae1d861a46..2a054c463a 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -1,10 +1,13 @@ -# encoding=utf-8 +# -*- coding: utf-8 -*- +""" +Import forms and submission utility functions. +""" import os import shutil import tempfile import zipfile -from builtins import open +from contextlib import ExitStack from django.core.files.uploadedfile import InMemoryUploadedFile from onadata.apps.logger.xform_fs import XFormInstanceFS @@ -30,9 +33,11 @@ def django_file(path, field_name, content_type): + """Returns an InMemoryUploadedFile object of a given file at the ``path``.""" # adapted from here: # http://groups.google.com/group/django-users/browse_thread/thread/ # 834f988876ff3c45/ + # pylint: disable=consider-using-with f = open(path, "rb") return InMemoryUploadedFile( file=f, @@ -44,50 +49,67 @@ def django_file(path, field_name, content_type): ) -def import_instance(username, xform_path, photos, osm_files, status, raise_exception): +# pylint: disable=too-many-arguments +def import_instance(username, xform_path, photos, osm_files, status): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ - with django_file( - xform_path, field_name="xml_file", content_type="text/xml" - ) as xml_file: - images = [ - django_file(jpg, field_name="image", content_type="image/jpeg") - for jpg in photos + with ExitStack() as stack: + submission_file = stack.enter_context(open(xform_path, "rb")) + xml_file = stack.enter_context( + InMemoryUploadedFile( + submission_file, + "xml_file", + xform_path, + "text/xml", + os.path.getsize(xform_path), + None, + ) + ) + _media_files = [ + (stack.enter_context(open(path, "rb")), path, "image/jpeg") + for path in photos ] - images += [ - django_file(osm, field_name="image", content_type="text/xml") - for osm in osm_files + _media_files += [ + (stack.enter_context(open(path, "rb")), path, "text/xml") + for path in osm_files ] - try: - instance = create_instance(username, xml_file, images, status) - except Exception as e: - if raise_exception: - raise e - - for i in images: - i.close() + media_files = [ + stack.enter_context( + InMemoryUploadedFile( + open_file, + "image", + path, + content_type, + os.path.getsize(path), + None, + ) + ) + for open_file, path, content_type in _media_files + ] + instance = create_instance(username, xml_file, media_files, status) if instance: return 1 - else: - return 0 + return 0 @app.task(ignore_result=True) def import_instance_async(username, xform_path, photos, osm_files, status): - import_instance(username, xform_path, photos, osm_files, status, False) + """An async alias to import_instance() function.""" + import_instance(username, xform_path, photos, osm_files, status) def iterate_through_instances( dirpath, callback, user=None, status="zip", is_async=False ): + """Iterate through all files and directories in the given ``dirpath``.""" total_file_count = 0 success_count = 0 errors = [] - for directory, subdirs, subfiles in os.walk(dirpath): + for directory, _subdirs, subfiles in os.walk(dirpath): for filename in subfiles: filepath = os.path.join(directory, filename) if XFormInstanceFS.is_valid_instance(filepath): @@ -99,10 +121,7 @@ def iterate_through_instances( ) success_count += 1 else: - try: - success_count += callback(xfxs) - except Exception as e: - errors.append("%s => %s" % (xfxs.filename, str(e))) + success_count += callback(xfxs) del xfxs total_file_count += 1 @@ -110,13 +129,13 @@ def iterate_through_instances( def import_instances_from_zip(zipfile_path, user, status="zip"): + """Unzips a zip file and imports submission instances from it.""" + temp_directory = tempfile.mkdtemp() try: - temp_directory = tempfile.mkdtemp() - zf = zipfile.ZipFile(zipfile_path) - - zf.extractall(temp_directory) + with zipfile.ZipFile(zipfile_path) as zip_file: + zip_file.extractall(temp_directory) except zipfile.BadZipfile as e: - errors = ["%s" % e] + errors = [f"{e}"] return 0, 0, errors else: return import_instances_from_path(temp_directory, user, status) @@ -125,13 +144,15 @@ def import_instances_from_zip(zipfile_path, user, status="zip"): def import_instances_from_path(path, user, status="zip", is_async=False): + """Process all submission instances in the given directory tree at ``path``.""" + def callback(xform_fs): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ import_instance( - user.username, xform_fs.path, xform_fs.photos, xform_fs.osm, status, True + user.username, xform_fs.path, xform_fs.photos, xform_fs.osm, status ) if is_async: diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 41f9a3dc4b..9325a29e0d 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -72,7 +72,8 @@ def _bad_request(e): def _extract_uuid(input_string): - input_string = input_string[input_string.find("@key=") : -1].replace("@key=", "") + key_index = input_string.find("@key=") + input_string = input_string[key_index:-1].replace("@key=", "") if input_string.startswith("uuid:"): input_string = input_string.replace("uuid:", "") return input_string @@ -197,7 +198,7 @@ def bulksubmission_form(request, username=None): # pylint: disable=invalid-name @require_GET -def formList(request, username): +def formList(request, username): # noqa N802 """ formList view, /formList OpenRosa Form Discovery API 1.0. """ @@ -259,7 +260,7 @@ def formList(request, username): # pylint: disable=invalid-name @require_GET -def xformsManifest(request, username, id_string): +def xformsManifest(request, username, id_string): # noqa N802 """ XFormManifest view, part of OpenRosa Form Discovery API 1.0. """ @@ -299,7 +300,7 @@ def xformsManifest(request, username, id_string): # pylint: disable=too-many-branches @require_http_methods(["HEAD", "POST"]) @csrf_exempt -def submission(request, username=None): +def submission(request, username=None): # noqa C901 """ Submission view, /submission of the OpenRosa Form Submission API 1.0. """ @@ -715,8 +716,8 @@ def view_download_submission(request, username): form_id = request.GET.get("formId", None) if not isinstance(form_id, six.string_types): return HttpResponseBadRequest() - - id_string = form_id[0 : form_id.find("[")] + last_index = form_id.find("[") + id_string = form_id[0:last_index] form_id_parts = form_id.split("/") if form_id_parts.__len__() < 2: return HttpResponseBadRequest() diff --git a/onadata/apps/logger/xform_fs.py b/onadata/apps/logger/xform_fs.py index 875f01e712..0bc337afe4 100644 --- a/onadata/apps/logger/xform_fs.py +++ b/onadata/apps/logger/xform_fs.py @@ -1,19 +1,30 @@ +# -*- coding: utf-8 -*- +""" +ODK Collect/Briefcase XForm instances folder traversal. +""" import os import glob import re -class XFormInstanceFS(object): +# pylint: disable=too-many-instance-attributes +class XFormInstanceFS: + """A class to traverse an ODK Collect/Briefcase XForm instances folder.""" + def __init__(self, filepath): self.path = filepath self.directory, self.filename = os.path.split(self.path) self.xform_id = re.sub(".xml", "", self.filename) + self._photos = [] + self._osm = [] + self._metadata_directory = "" + self._xml = "" @property def photos(self): - if not hasattr(self, "_photos"): + """Returns all .jpg file paths.""" + if not getattr(self, "_photos"): available_photos = glob.glob(os.path.join(self.directory, "*.jpg")) - self._photos = [] for photo_path in available_photos: _pdir, photo = os.path.split(photo_path) if self.xml.find(photo) > 0: @@ -22,9 +33,9 @@ def photos(self): @property def osm(self): - if not hasattr(self, "_osm"): + """Returns all .osm file paths.""" + if not getattr(self, "_osm"): available_osm = glob.glob(os.path.join(self.directory, "*.osm")) - self._osm = [] for osm_path in available_osm: _pdir, osm = os.path.split(osm_path) if self.xml.find(osm) > 0: @@ -33,31 +44,32 @@ def osm(self): @property def metadata_directory(self): - if not hasattr(self, "_metadata_directory"): + """Returns the metadata directory.""" + if not getattr(self, "_metadata_directory"): instances_dir = os.path.join(self.directory, "..", "..", "instances") metadata_directory = os.path.join(self.directory, "..", "..", "metadata") if os.path.exists(instances_dir) and os.path.exists(metadata_directory): self._metadata_directory = os.path.abspath(metadata_directory) - else: - self._metadata_directory = False return self._metadata_directory @property def xml(self): - if not hasattr(self, "_xml"): - with open(self.path, "r") as f: + """Returns the submission XML""" + if not getattr(self, "_xml"): + with open(self.path, "r", encoding="utf-8") as f: self._xml = f.read() return self._xml @classmethod def is_valid_instance(cls, filepath): + """Returns True if the XML at ``filepath`` is a valid XML file.""" if not filepath.endswith(".xml"): return False - with open(filepath, "r") as ff: - fxml = ff.read() + with open(filepath, "r", encoding="utf-8") as xml_file: + fxml = xml_file.read() if not fxml.strip().startswith("" % self.xform_id + return f"" diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index b12a2319d4..bf2e160049 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -174,7 +174,7 @@ def clean_and_parse_xml(xml_string): # pylint: disable=too-many-branches -def _xml_node_to_dict(node, repeats=None, encrypted=False): +def _xml_node_to_dict(node, repeats=None, encrypted=False): # noqa C901 repeats = [] if repeats is None else repeats if len(node.childNodes) == 0: # there's no data for this leaf node From efdc717b2aa61f1183fb8d870b29774d4f012893 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 12 May 2022 20:58:34 +0300 Subject: [PATCH 206/234] logger: cleanup --- .../management/commands/create_backup.py | 39 ++++++++++++------- onadata/apps/logger/models/attachment.py | 38 ++++++++++++------ 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/onadata/apps/logger/management/commands/create_backup.py b/onadata/apps/logger/management/commands/create_backup.py index a6a54fde72..9a05c5830f 100644 --- a/onadata/apps/logger/management/commands/create_backup.py +++ b/onadata/apps/logger/management/commands/create_backup.py @@ -1,36 +1,47 @@ +# -*- coding: utf-8 -*- +""" +create_backup - command to create zipped backup of a form and it's submissions. +""" import os +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import gettext_lazy, gettext as _ -from django.contrib.auth.models import User +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm from onadata.libs.utils.backup_tools import create_zip_backup class Command(BaseCommand): + """Create a zip backup of a form and all its submissions.""" + args = "outfile username [id_string]" help = gettext_lazy("Create a zip backup of a form and all its submissions") - def handle(self, *args, **options): + # pylint: disable=unused-argument + def handle(self, *args, **options): # noqa C901 + """Create a zip backup of a form and all its submissions.""" try: output_file = args[0] - except IndexError: - raise CommandError(_("Provide the path to the zip file to backup" " to")) + except IndexError as e: + raise CommandError( + _("Provide the path to the zip file to backup to") + ) from e else: output_file = os.path.realpath(output_file) try: username = args[1] - except IndexError: + except IndexError as e: raise CommandError( - _("You must provide the username to publish the" " form to.") - ) + _("You must provide the username to publish the form to.") + ) from e # make sure user exists try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + user = get_user_model().objects.get(username=username) + except get_user_model().DoesNotExist as e: + raise CommandError(_(f"The user '{username}' does not exist.")) from e try: id_string = args[2] @@ -40,6 +51,8 @@ def handle(self, *args, **options): # make sure xform exists try: xform = XForm.objects.get(user=user, id_string=id_string) - except XForm.DoesNotExist: - raise CommandError(_("The id_string '%s' does not exist.") % id_string) + except XForm.DoesNotExist as e: + raise CommandError( + _(f"The id_string '{id_string}' does not exist.") + ) from e create_zip_backup(output_file, user, xform) diff --git a/onadata/apps/logger/models/attachment.py b/onadata/apps/logger/models/attachment.py index 3e254fd222..5a58a186b3 100644 --- a/onadata/apps/logger/models/attachment.py +++ b/onadata/apps/logger/models/attachment.py @@ -1,12 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Attachment model. +""" +import hashlib import mimetypes import os -from hashlib import md5 +from django.contrib.auth import get_user_model from django.db import models -from django.contrib.auth.models import User def get_original_filename(filename): + """Returns the filename removing the hashed random string added to it when we have + file duplicates in some file systems.""" # https://docs.djangoproject.com/en/1.8/ref/files/storage/ # #django.core.files.storage.Storage.get_available_name # If a file with name already exists, an underscore plus a random @@ -26,9 +32,9 @@ def get_original_filename(filename): def upload_to(instance, filename): - folder = "{}_{}".format( - instance.instance.xform.id, instance.instance.xform.id_string - ) + """Returns the attachments folder to upload the file to.""" + folder = f"{instance.instance.xform.id}_{instance.instance.xform.id_string}" + return os.path.join( instance.instance.xform.user.username, "attachments", @@ -58,7 +64,10 @@ class Attachment(models.Model): file_size = models.PositiveIntegerField(default=0) name = models.CharField(max_length=100, null=True, blank=True) deleted_by = models.ForeignKey( - User, related_name="deleted_attachments", null=True, on_delete=models.SET_NULL + get_user_model(), + related_name="deleted_attachments", + null=True, + on_delete=models.SET_NULL, ) class Meta: @@ -68,7 +77,7 @@ class Meta: def save(self, *args, **kwargs): if self.media_file and self.mimetype == "": # guess mimetype - mimetype, encoding = mimetypes.guess_type(self.media_file.name) + mimetype, _encoding = mimetypes.guess_type(self.media_file.name) if mimetype: self.mimetype = mimetype if self.media_file and len(self.media_file.name) > 255: @@ -81,15 +90,22 @@ def save(self, *args, **kwargs): except (OSError, AttributeError): pass - super(Attachment, self).save(*args, **kwargs) + super().save(*args, **kwargs) @property def file_hash(self): + """Returns the MD5 hash of the file.""" + md5_hash = "" if self.media_file.storage.exists(self.media_file.name): - return "%s" % md5(self.media_file.read()).hexdigest() - return "" + md5_hash = hashlib.new( + "md5", self.media_file.read(), usedforsecurity=False + ).hexdigest() + return md5_hash @property def filename(self): + """Returns the attachment's filename.""" + filename = "" if self.media_file: - return os.path.basename(self.media_file.name) + filename = os.path.basename(self.media_file.name) + return filename From 8b54914c437192068953e232acc55b22bf9567f8 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 12 May 2022 21:35:26 +0300 Subject: [PATCH 207/234] import_tools.py: handle silently Http404 exception --- onadata/apps/logger/import_tools.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index 2a054c463a..f25d5b8a71 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -6,9 +6,10 @@ import shutil import tempfile import zipfile - from contextlib import ExitStack + from django.core.files.uploadedfile import InMemoryUploadedFile +from django.http.response import Http404 from onadata.apps.logger.xform_fs import XFormInstanceFS from onadata.celery import app @@ -121,7 +122,13 @@ def iterate_through_instances( ) success_count += 1 else: - success_count += callback(xfxs) + try: + count = callback(xfxs) + except Http404: + pass + else: + if count: + success_count += count del xfxs total_file_count += 1 From 5445c540c43c4c5c9545cbee29790b739812be09 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 6 May 2022 15:09:05 +0300 Subject: [PATCH 208/234] Add `google-auth-oauthlib` requirement --- requirements/base.pip | 22 +++++++++++++++++++--- setup.cfg | 2 ++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index 673d92cdf5..4c3bb879f5 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -48,6 +48,8 @@ botocore==1.25.7 # s3transfer cached-property==1.5.2 # via tableschema +cachetools==5.0.0 + # via google-auth celery==5.2.6 # via onadata certifi==2021.10.8 @@ -179,6 +181,10 @@ future==0.18.2 # via python-json2xlsclient geojson==2.5.0 # via onadata +google-auth==2.6.6 + # via google-auth-oauthlib +google-auth-oauthlib==0.5.1 + # via onadata greenlet==1.1.2 # via sqlalchemy httmock==1.4.0 @@ -242,7 +248,9 @@ nose==1.3.7 numpy==1.22.3 # via onadata oauthlib==3.2.0 - # via django-oauth-toolkit + # via + # django-oauth-toolkit + # requests-oauthlib openpyxl==3.0.9 # via # onadata @@ -268,7 +276,9 @@ pyasn1==0.4.8 # pyasn1-modules # rsa pyasn1-modules==0.2.8 - # via oauth2client + # via + # google-auth + # oauth2client pycodestyle==2.8.0 # via flake8 pycparser==2.21 @@ -329,15 +339,20 @@ requests==2.27.1 # onadata # python-json2xlsclient # requests-mock + # requests-oauthlib # sphinx # tableschema # tabulator requests-mock==1.9.3 # via onadata +requests-oauthlib==1.3.1 + # via google-auth-oauthlib rfc3986==2.0.0 # via tableschema rsa==4.8 - # via oauth2client + # via + # google-auth + # oauth2client s3transfer==0.5.2 # via boto3 savreaderwriter==3.4.2 @@ -352,6 +367,7 @@ six==1.16.0 # datapackage # django-query-builder # djangorestframework-csv + # google-auth # isodate # linear-tsv # oauth2client diff --git a/setup.cfg b/setup.cfg index be16db420f..3aa28e889f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,6 +105,8 @@ install_requires = django-redis # osm defusedxml + # Google exports + google-auth-oauthlib python_requires = >= 3.9 setup_requires = setuptools_scm From 985f146d56ee0e48d83a6bd71cbe3e4cbea67225 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 6 May 2022 15:30:26 +0300 Subject: [PATCH 209/234] Use Flow class from `google_auth_oauthlib` --- onadata/libs/utils/google.py | 25 ++++++++++++++++--------- onadata/settings/common.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index cb0fb7c0cd..157433da26 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -1,11 +1,18 @@ -from oauth2client.client import OAuth2WebServerFlow +from typing import Optional + from django.conf import settings +from google_auth_oauthlib.flow import Flow + -google_flow = OAuth2WebServerFlow( - client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, - client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=' '.join( - ['https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', - 'https://www.googleapis.com/auth/drive.file']), - redirect_uri=settings.GOOGLE_STEP2_URI, prompt="consent") +def create_flow(redirect_uri: Optional[str] = None) -> Flow: + return Flow.from_client_config( + settings.GOOGLE_FLOW, + scopes=['https://docs.google.com/feeds/', + 'https://spreadsheets.google.com/feeds/', + 'https://www.googleapis.com/auth/drive.file'], + redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI, prompt="consent") +# google_flow = OAuth2WebServerFlow( +# client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, +# client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, +# scope=' '.join( +# redirect_uri=settings.GOOGLE_STEP2_URI, prompt="consent") diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 97f7764c68..43665539a2 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -592,3 +592,17 @@ def configure_logging(logger, **kwargs): XFORM_CHARTS_CACHE_TIME = 600 SLAVE_DATABASES = [] + +# Google Export settings +GOOGLE_FLOW = { + "web": { + "client_id": "", + "project_id": "", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "", + "redirect_uris": [], + "javascript_origins": [] + } +} From 6bc459cc77bba2a5b8e118a88aa63947a53c62fc Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 6 May 2022 15:52:35 +0300 Subject: [PATCH 210/234] Disable credential storage and utilize `create_flow` function --- onadata/apps/viewer/views.py | 25 +++++----- onadata/libs/utils/api_export_tools.py | 68 ++++++++++---------------- 2 files changed, 39 insertions(+), 54 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index cf3dd20f91..c91b449e9d 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -31,8 +31,6 @@ import requests from dict2xml import dict2xml from dpath import util as dpath_util -from oauth2client import client as google_client -from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage try: from savReaderWriter import SPSSIOError @@ -57,7 +55,7 @@ should_create_new_export, str_to_bool, ) -from onadata.libs.utils.google import google_flow +from onadata.libs.utils.google import create_flow from onadata.libs.utils.image_tools import image_url from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import ( @@ -471,14 +469,16 @@ def create_export(request, username, id_string, export_type): def _get_google_credential(request): - token = None - if request.user.is_authenticated: - storage = Storage(TokenStorageModel, "id", request.user, "credential") - credential = storage.get() - elif request.session.get("access_token"): - credential = google_client.OAuth2Credentials.from_json(token) - - return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) + # token = None + # if request.user.is_authenticated: + # storage = Storage(TokenStorageModel, "id", request.user, "credential") + # credential = storage.get() + # elif request.session.get("access_token"): + # credential = google_client.OAuth2Credentials.from_json(token) + # + # return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) + google_flow = create_flow() + return HttpResponseRedirect(google_flow.authorization_url()) def export_list(request, username, id_string, export_type): # noqa C901 @@ -805,7 +805,8 @@ def google_xls_export(request, username, id_string): request.session["google_redirect_url"] = reverse( google_xls_export, kwargs={"username": username, "id_string": id_string} ) - return HttpResponseRedirect(google_flow.step1_get_authorize_url()) + google_flow = create_flow() + return HttpResponseRedirect(google_flow.authorization_url()) owner = get_object_or_404(User, username__iexact=username) xform = get_form({"user": owner, "id_string__iexact": id_string}) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index ca6b8f3cf3..91aed817c8 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -16,13 +16,6 @@ from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ from kombu.exceptions import OperationalError -from oauth2client import client as google_client -from oauth2client.client import ( - HttpAccessTokenRefreshError, - OAuth2WebServerFlow, - TokenRevokeError, -) -from oauth2client.contrib.django_util.storage import DjangoORMStorage as Storage from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse @@ -43,6 +36,7 @@ ) from onadata.libs.permissions import filter_queryset_xform_meta_perms_sql from onadata.libs.utils import log +from onadata.libs.utils.google import create_flow from onadata.libs.utils.async_status import ( FAILED, PENDING, @@ -596,40 +590,30 @@ def generate_google_web_flow(request): redirect_uri = request.data.get("redirect_uri") else: redirect_uri = settings.GOOGLE_STEP2_URI - return OAuth2WebServerFlow( - client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, - client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, - scope=" ".join( - [ - "https://docs.google.com/feeds/", - "https://spreadsheets.google.com/feeds/", - "https://www.googleapis.com/auth/drive.file", - ] - ), - redirect_uri=redirect_uri, - prompt="consent", - ) - + return create_flow(redirect_uri) def _get_google_credential(request): - token = None - credential = None - if request.user.is_authenticated: - storage = Storage(TokenStorageModel, "id", request.user, "credential") - credential = storage.get() - elif request.session.get("access_token"): - credential = google_client.OAuth2Credentials.from_json(token) - - if credential: - try: - credential.get_access_token() - except HttpAccessTokenRefreshError: - try: - credential.revoke(httplib2.Http()) - except TokenRevokeError: - storage.delete() - - if not credential or credential.invalid: - google_flow = generate_google_web_flow(request) - return HttpResponseRedirect(google_flow.step1_get_authorize_url()) - return credential + # TODO: Handle storage and retrieval + # token = None + # credential = None + # if request.user.is_authenticated: + # storage = Storage(TokenStorageModel, "id", request.user, "credential") + # credential = storage.get() + # elif request.session.get("access_token"): + # credential = google_client.OAuth2Credentials.from_json(token) + + # if credential: + # try: + # credential.get_access_token() + # except HttpAccessTokenRefreshError: + # try: + # credential.revoke(httplib2.Http()) + # except TokenRevokeError: + # storage.delete() + + # if not credential or credential.invalid: + # google_flow = generate_google_web_flow(request) + # return HttpResponseRedirect(google_flow.step1_get_authorize_url()) + # return credential + google_flow = generate_google_web_flow(request) + return HttpResponseRedirect(google_flow.authorization_url()) From 25424c0a02e46f6d92718e0f84458bda728d51d2 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 6 May 2022 16:50:57 +0300 Subject: [PATCH 211/234] Add `google-auth` requirement --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 3aa28e889f..fb2e3da22c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -107,6 +107,7 @@ install_requires = defusedxml # Google exports google-auth-oauthlib + google-auth python_requires = >= 3.9 setup_requires = setuptools_scm From f9d314ce36fe59a78f63004a0274f7b2bee83619 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 6 May 2022 16:57:23 +0300 Subject: [PATCH 212/234] Add `CredentialsField` class --- onadata/apps/main/models/google_oath.py | 67 ++++++++++++++++++++++++- requirements/base.pip | 6 ++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/onadata/apps/main/models/google_oath.py b/onadata/apps/main/models/google_oath.py index 46df88fc01..6a39388f55 100644 --- a/onadata/apps/main/models/google_oath.py +++ b/onadata/apps/main/models/google_oath.py @@ -2,9 +2,74 @@ """ Google auth token storage model class """ +import base64 +import pickle +import jsonpickle + from django.conf import settings from django.db import models -from oauth2client.contrib.django_util.models import CredentialsField +from django.utils import encoding + +from google.auth.credentials import Credentials + + +class CredentialsField(models.Field): + """ + Django ORM field for storing OAuth2 Credentials. + Modified version of + https://github.com/onaio/oauth2client/blob/master/oauth2client/contrib/django_util/models.py + """ + + def __init__(self, *args, **kwargs): + if 'null' not in kwargs: + kwargs['null'] = True + super(CredentialsField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return 'BinaryField' + + def from_db_value(self, value, expression, connection, context): + """Overrides ``models.Field`` method. This converts the value + returned from the database to an instance of this class. + """ + return self.to_python(value) + + def to_python(self, value): + """Overrides ``models.Field`` method. This is used to convert + bytes (from serialization etc) to an instance of this class""" + if value is None: + return None + elif isinstance(value, Credentials): + return value + else: + try: + return jsonpickle.decode( + base64.b64decode(encoding.smart_bytes(value)).decode()) + except ValueError: + return pickle.loads( + base64.b64decode(encoding.smart_bytes(value))) + + def get_prep_value(self, value): + """Overrides ``models.Field`` method. This is used to convert + the value from an instances of this class to bytes that can be + inserted into the database. + """ + if value is None: + return None + else: + return encoding.smart_text( + base64.b64encode(jsonpickle.encode(value).encode())) + + def value_to_string(self, obj): + """Convert the field value from the provided model to a string. + Used during model serialization. + Args: + obj: db.Model, model object + Returns: + string, the serialized field value + """ + value = self.value_from_object(obj) + return self.get_prep_value(value) class TokenStorageModel(models.Model): diff --git a/requirements/base.pip b/requirements/base.pip index 4c3bb879f5..ad40032c69 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -12,6 +12,8 @@ # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in +-e file:///home/dave/src/onadata + # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in -e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest @@ -182,7 +184,9 @@ future==0.18.2 geojson==2.5.0 # via onadata google-auth==2.6.6 - # via google-auth-oauthlib + # via + # google-auth-oauthlib + # onadata google-auth-oauthlib==0.5.1 # via onadata greenlet==1.1.2 From b7884b7a06f21bbc49fb86283df1c495a814a849 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 10:14:11 +0300 Subject: [PATCH 213/234] Remove unsupported `prompt` keyword arguement --- onadata/libs/utils/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index 157433da26..80e759fd47 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -10,7 +10,7 @@ def create_flow(redirect_uri: Optional[str] = None) -> Flow: scopes=['https://docs.google.com/feeds/', 'https://spreadsheets.google.com/feeds/', 'https://www.googleapis.com/auth/drive.file'], - redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI, prompt="consent") + redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI) # google_flow = OAuth2WebServerFlow( # client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, # client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, From f414cce977d3f06c406694da5114cadba8bfd4e6 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 10:45:15 +0300 Subject: [PATCH 214/234] Remove local directory path --- requirements/base.pip | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index ad40032c69..4d8f81a427 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -12,8 +12,6 @@ # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in --e file:///home/dave/src/onadata - # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in -e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest From 6fc11cddab4b5a166e3e5a2baf716fe6b8c8f211 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 14:37:02 +0300 Subject: [PATCH 215/234] Remove oauth2client requirement --- requirements/base.in | 3 +-- requirements/base.pip | 16 +++------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 5b9f879bbf..c2732842d0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -7,5 +7,4 @@ -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client --e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc \ No newline at end of file +-e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc diff --git a/requirements/base.pip b/requirements/base.pip index 4d8f81a427..04a709a010 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -8,8 +8,6 @@ # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router # via -r requirements/base.in --e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client - # via -r requirements/base.in -e git+https://github.com/onaio/ona-oidc.git@v0.0.10#egg=ona-oidc # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip @@ -192,9 +190,7 @@ greenlet==1.1.2 httmock==1.4.0 # via onadata httplib2==0.20.4 - # via - # oauth2client - # onadata + # via onadata idna==3.3 # via requests ijson==3.1.4 @@ -274,13 +270,10 @@ psycopg2-binary==2.9.3 # via onadata pyasn1==0.4.8 # via - # oauth2client # pyasn1-modules # rsa pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client + # via google-auth pycodestyle==2.8.0 # via flake8 pycparser==2.21 @@ -352,9 +345,7 @@ requests-oauthlib==1.3.1 rfc3986==2.0.0 # via tableschema rsa==4.8 - # via - # google-auth - # oauth2client + # via google-auth s3transfer==0.5.2 # via boto3 savreaderwriter==3.4.2 @@ -372,7 +363,6 @@ six==1.16.0 # google-auth # isodate # linear-tsv - # oauth2client # python-dateutil # python-memcached # requests-mock From 524a576e7ccc1d5abb71981aaeffae14b0bd10b3 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 14:39:34 +0300 Subject: [PATCH 216/234] Update `from_db_value` function signature - Use OAuth2 credentials instead of regular credentials class --- onadata/apps/main/models/google_oath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/main/models/google_oath.py b/onadata/apps/main/models/google_oath.py index 6a39388f55..d40dd70d15 100644 --- a/onadata/apps/main/models/google_oath.py +++ b/onadata/apps/main/models/google_oath.py @@ -10,7 +10,7 @@ from django.db import models from django.utils import encoding -from google.auth.credentials import Credentials +from google.oauth2.credentials import Credentials class CredentialsField(models.Field): @@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs): def get_internal_type(self): return 'BinaryField' - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection, context=None): """Overrides ``models.Field`` method. This converts the value returned from the database to an instance of this class. """ From 4f01f789502189784e39a0a9f8ff06aed63b834f Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 14:40:28 +0300 Subject: [PATCH 217/234] Use correct scopes when creating flow object --- onadata/libs/utils/google.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index 80e759fd47..67ee762945 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -7,10 +7,11 @@ def create_flow(redirect_uri: Optional[str] = None) -> Flow: return Flow.from_client_config( settings.GOOGLE_FLOW, - scopes=['https://docs.google.com/feeds/', - 'https://spreadsheets.google.com/feeds/', + scopes=['https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/docs', 'https://www.googleapis.com/auth/drive.file'], redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI) + # google_flow = OAuth2WebServerFlow( # client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, # client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, From 4b0543c11494c2340061764d1372b0fce1e753d3 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 14:40:43 +0300 Subject: [PATCH 218/234] Handle storage and retrieval of OAuth2 credentials --- onadata/libs/utils/api_export_tools.py | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 91aed817c8..274bdb8717 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -593,27 +593,21 @@ def generate_google_web_flow(request): return create_flow(redirect_uri) def _get_google_credential(request): - # TODO: Handle storage and retrieval - # token = None - # credential = None - # if request.user.is_authenticated: - # storage = Storage(TokenStorageModel, "id", request.user, "credential") - # credential = storage.get() - # elif request.session.get("access_token"): - # credential = google_client.OAuth2Credentials.from_json(token) - - # if credential: - # try: - # credential.get_access_token() - # except HttpAccessTokenRefreshError: - # try: - # credential.revoke(httplib2.Http()) - # except TokenRevokeError: - # storage.delete() + token = None + credential = None + if request.user.is_authenticated: + storage = TokenStorageModel.objects.filter(id=request.user) + credential = storage.first() + # elif request.session.get("access_token"): + # credential = google_client.OAuth2Credentials.from_json(token) + + if credential: + credential.get_access_token() # if not credential or credential.invalid: # google_flow = generate_google_web_flow(request) # return HttpResponseRedirect(google_flow.step1_get_authorize_url()) # return credential google_flow = generate_google_web_flow(request) - return HttpResponseRedirect(google_flow.authorization_url()) + authorization_url, state = google_flow.authorization_url() + return HttpResponseRedirect(authorization_url) From de25395cf9aae32c0c6dc2ab892f7e1871c77068 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 16:29:23 +0300 Subject: [PATCH 219/234] Use internal `CredentialsField` class --- onadata/apps/main/migrations/0007_auto_20160418_0525.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/onadata/apps/main/migrations/0007_auto_20160418_0525.py b/onadata/apps/main/migrations/0007_auto_20160418_0525.py index d2f4dc5311..5778929ea9 100644 --- a/onadata/apps/main/migrations/0007_auto_20160418_0525.py +++ b/onadata/apps/main/migrations/0007_auto_20160418_0525.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals from django.db import migrations -import oauth2client.contrib.django_util.models +from onadata.apps.main.models.google_oath import CredentialsField class Migration(migrations.Migration): @@ -20,7 +20,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='tokenstoragemodel', name='credential', - field=oauth2client.contrib.django_util.models.CredentialsField( - null=True), + field=CredentialsField(null=True), ), ] From 18d3a0c304c5c7b31b609877aa9445031b4f6c09 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 9 May 2022 16:30:06 +0300 Subject: [PATCH 220/234] Implement authorization URL best practices - Format files Read more: https://developers.google.com/identity/protocols/oauth2/web-server#creatingclient --- onadata/apps/viewer/views.py | 14 +++++-------- onadata/libs/utils/api_export_tools.py | 29 ++++++++++++-------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index c91b449e9d..f3d0449e73 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -469,16 +469,12 @@ def create_export(request, username, id_string, export_type): def _get_google_credential(request): - # token = None - # if request.user.is_authenticated: - # storage = Storage(TokenStorageModel, "id", request.user, "credential") - # credential = storage.get() - # elif request.session.get("access_token"): - # credential = google_client.OAuth2Credentials.from_json(token) - # - # return credential or HttpResponseRedirect(google_flow.step1_get_authorize_url()) google_flow = create_flow() - return HttpResponseRedirect(google_flow.authorization_url()) + return HttpResponseRedirect( + google_flow.authorization_url( + access_type="offline", include_granted_scopes="true" + ) + ) def export_list(request, username, id_string, export_type): # noqa C901 diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 274bdb8717..d0db3ba597 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -592,22 +592,19 @@ def generate_google_web_flow(request): redirect_uri = settings.GOOGLE_STEP2_URI return create_flow(redirect_uri) + def _get_google_credential(request): - token = None credential = None if request.user.is_authenticated: - storage = TokenStorageModel.objects.filter(id=request.user) - credential = storage.first() - # elif request.session.get("access_token"): - # credential = google_client.OAuth2Credentials.from_json(token) - - if credential: - credential.get_access_token() - - # if not credential or credential.invalid: - # google_flow = generate_google_web_flow(request) - # return HttpResponseRedirect(google_flow.step1_get_authorize_url()) - # return credential - google_flow = generate_google_web_flow(request) - authorization_url, state = google_flow.authorization_url() - return HttpResponseRedirect(authorization_url) + try: + storage = TokenStorageModel.objects.get(id=request.user) + credential = storage.credential + except TokenStorageModel.DoesNotExist: + pass + + if not credential: + google_flow = generate_google_web_flow(request) + authorization_url, state = google_flow.authorization_url( + access_type="offline", include_granted_scopes="true" + ) + return HttpResponseRedirect(authorization_url) From a3583dd33544904c8b9a474370ca341b5c5eb064 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Tue, 10 May 2022 12:19:41 +0300 Subject: [PATCH 221/234] Ensure credential is returned if present --- onadata/libs/utils/api_export_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index d0db3ba597..ffb7284d14 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -608,3 +608,4 @@ def _get_google_credential(request): access_type="offline", include_granted_scopes="true" ) return HttpResponseRedirect(authorization_url) + return credential From 3f8ffb473d5c34ae8874fb951ddf3c90c074e209 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Tue, 10 May 2022 13:29:45 +0300 Subject: [PATCH 222/234] Code cleanup --- onadata/libs/utils/google.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index 67ee762945..680aa00f04 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -11,9 +11,3 @@ def create_flow(redirect_uri: Optional[str] = None) -> Flow: 'https://www.googleapis.com/auth/docs', 'https://www.googleapis.com/auth/drive.file'], redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI) - -# google_flow = OAuth2WebServerFlow( -# client_id=settings.GOOGLE_OAUTH2_CLIENT_ID, -# client_secret=settings.GOOGLE_OAUTH2_CLIENT_SECRET, -# scope=' '.join( -# redirect_uri=settings.GOOGLE_STEP2_URI, prompt="consent") From 6a0927ac4f9a7201bef4b81da96f8db7ece15571 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Tue, 10 May 2022 13:45:22 +0300 Subject: [PATCH 223/234] Format code --- onadata/libs/utils/api_export_tools.py | 1 - onadata/libs/utils/google.py | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index ffb7284d14..e43050fdf2 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -8,7 +8,6 @@ from datetime import datetime import six -import httplib2 from celery.backends.rpc import BacklogLimitExceeded from celery.result import AsyncResult from django.conf import settings diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index 680aa00f04..fb4512de46 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -7,7 +7,10 @@ def create_flow(redirect_uri: Optional[str] = None) -> Flow: return Flow.from_client_config( settings.GOOGLE_FLOW, - scopes=['https://www.googleapis.com/auth/spreadsheets', - 'https://www.googleapis.com/auth/docs', - 'https://www.googleapis.com/auth/drive.file'], - redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI) + scopes=[ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/docs", + "https://www.googleapis.com/auth/drive.file", + ], + redirect_uri=redirect_uri or settings.GOOGLE_STEP2_URI, + ) From c74304b37bd0d3d9a64c37abf1faab98bd7d6e5d Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Wed, 11 May 2022 13:14:44 +0300 Subject: [PATCH 224/234] Prompt for consent everytime we authenticate with google --- onadata/apps/viewer/views.py | 8 ++++++-- onadata/libs/utils/api_export_tools.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index f3d0449e73..76ca8fc6b9 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -472,7 +472,7 @@ def _get_google_credential(request): google_flow = create_flow() return HttpResponseRedirect( google_flow.authorization_url( - access_type="offline", include_granted_scopes="true" + access_type="offline", include_granted_scopes="true", prompt="consent" ) ) @@ -802,7 +802,11 @@ def google_xls_export(request, username, id_string): google_xls_export, kwargs={"username": username, "id_string": id_string} ) google_flow = create_flow() - return HttpResponseRedirect(google_flow.authorization_url()) + return HttpResponseRedirect( + google_flow.authorization_url( + access_type="offline", include_granted_scopes="true", prompt="consent" + ) + ) owner = get_object_or_404(User, username__iexact=username) xform = get_form({"user": owner, "id_string__iexact": id_string}) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index e43050fdf2..61b34710e0 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -604,7 +604,7 @@ def _get_google_credential(request): if not credential: google_flow = generate_google_web_flow(request) authorization_url, state = google_flow.authorization_url( - access_type="offline", include_granted_scopes="true" + access_type="offline", include_granted_scopes="true", prompt="consent" ) return HttpResponseRedirect(authorization_url) return credential From 18ba7669b85fe23b947cf1f5bcb5b966819334ad Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 13 May 2022 10:44:19 +0300 Subject: [PATCH 225/234] Add migration to convert oauth2client.Credentials class --- .../migrations/0011_auto_20220510_0907.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 onadata/apps/main/migrations/0011_auto_20220510_0907.py diff --git a/onadata/apps/main/migrations/0011_auto_20220510_0907.py b/onadata/apps/main/migrations/0011_auto_20220510_0907.py new file mode 100644 index 0000000000..1bcd5cc6da --- /dev/null +++ b/onadata/apps/main/migrations/0011_auto_20220510_0907.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.13 on 2022-05-10 13:07 +from django.db import migrations + +from google.oauth2.credentials import Credentials +from onadata.apps.main.models.google_oath import TokenStorageModel + + +def convert_oauth2client_credentials(apps, schema_editor): + """ + Converts the OAuth2Client Credentials class object + into a google.oauth2.credentials.Credentials object + """ + queryset = TokenStorageModel.objects.all() + for storage in queryset.iterator(): + if ( + isinstance(storage.credential, dict) + and "py/state" in storage.credential.keys() + ): + data = storage.credential.get("py/state") + scopes = data.pop("scopes") + if isinstance(scopes, dict): + scopes = scopes.get("py/set") + credential = Credentials.from_authorized_user_info(data, scopes=scopes) + storage.credential = credential + storage.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("main", "0010_auto_20220425_0313"), + ] + + operations = [migrations.RunPython(convert_oauth2client_credentials)] From 97c2d1c77e593a7e69b93c0cb5e4e30791714d3b Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 13 May 2022 11:25:57 +0300 Subject: [PATCH 226/234] Create credentials using access token when present --- onadata/libs/utils/api_export_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 61b34710e0..c23ec896c1 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -18,6 +18,7 @@ from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse +from google.oauth2.credentials import Credentials try: from savReaderWriter import SPSSIOError @@ -600,6 +601,8 @@ def _get_google_credential(request): credential = storage.credential except TokenStorageModel.DoesNotExist: pass + elif request.session.get("access_token"): + credential = Credentials(token=request.session["access_token"]) if not credential: google_flow = generate_google_web_flow(request) From c6d7c01bfaaa6b6c08bf128015cf1461d22eaaf8 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 13 May 2022 13:49:29 +0300 Subject: [PATCH 227/234] Google: cleanup --- onadata/apps/main/models/google_oath.py | 42 +++++++++++++------------ onadata/libs/utils/api_export_tools.py | 8 +++-- onadata/libs/utils/google.py | 6 ++++ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/onadata/apps/main/models/google_oath.py b/onadata/apps/main/models/google_oath.py index d40dd70d15..b74b64cf37 100644 --- a/onadata/apps/main/models/google_oath.py +++ b/onadata/apps/main/models/google_oath.py @@ -4,12 +4,12 @@ """ import base64 import pickle -import jsonpickle -from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models from django.utils import encoding +import jsonpickle from google.oauth2.credentials import Credentials @@ -21,13 +21,14 @@ class CredentialsField(models.Field): """ def __init__(self, *args, **kwargs): - if 'null' not in kwargs: - kwargs['null'] = True - super(CredentialsField, self).__init__(*args, **kwargs) + if "null" not in kwargs: + kwargs["null"] = True + super().__init__(*args, **kwargs) def get_internal_type(self): - return 'BinaryField' + return "BinaryField" + # pylint: disable=unused-argument def from_db_value(self, value, expression, connection, context=None): """Overrides ``models.Field`` method. This converts the value returned from the database to an instance of this class. @@ -39,15 +40,14 @@ def to_python(self, value): bytes (from serialization etc) to an instance of this class""" if value is None: return None - elif isinstance(value, Credentials): + if isinstance(value, Credentials): return value - else: - try: - return jsonpickle.decode( - base64.b64decode(encoding.smart_bytes(value)).decode()) - except ValueError: - return pickle.loads( - base64.b64decode(encoding.smart_bytes(value))) + try: + return jsonpickle.decode( + base64.b64decode(encoding.smart_bytes(value)).decode() + ) + except ValueError: + return pickle.loads(base64.b64decode(encoding.smart_bytes(value))) def get_prep_value(self, value): """Overrides ``models.Field`` method. This is used to convert @@ -56,9 +56,7 @@ def get_prep_value(self, value): """ if value is None: return None - else: - return encoding.smart_text( - base64.b64encode(jsonpickle.encode(value).encode())) + return encoding.smart_str(base64.b64encode(jsonpickle.encode(value).encode())) def value_to_string(self, obj): """Convert the field value from the provided model to a string. @@ -77,10 +75,14 @@ class TokenStorageModel(models.Model): Google Auth Token storage model """ + # pylint: disable=invalid-name id = models.OneToOneField( - settings.AUTH_USER_MODEL, primary_key=True, related_name='google_id', - on_delete=models.CASCADE) + get_user_model(), + primary_key=True, + related_name="google_id", + on_delete=models.CASCADE, + ) credential = CredentialsField() class Meta: - app_label = 'main' + app_label = "main" diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index c23ec896c1..777833495a 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -108,7 +108,7 @@ def _get_export_type(export_type): # pylint: disable=too-many-arguments, too-many-locals, too-many-branches -def custom_response_handler( +def custom_response_handler( # noqa: C0901 request, xform, query, @@ -216,7 +216,9 @@ def _new_export(): return response -def _generate_new_export(request, xform, query, export_type, dataview_pk=False): +def _generate_new_export( # noqa: C0901 + request, xform, query, export_type, dataview_pk=False +): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) @@ -606,7 +608,7 @@ def _get_google_credential(request): if not credential: google_flow = generate_google_web_flow(request) - authorization_url, state = google_flow.authorization_url( + authorization_url, _state = google_flow.authorization_url( access_type="offline", include_granted_scopes="true", prompt="consent" ) return HttpResponseRedirect(authorization_url) diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index fb4512de46..33fb4cd5f9 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -1,10 +1,16 @@ +# -*- coding=utf-8 -*- +""" +Google utility functions. +""" from typing import Optional from django.conf import settings + from google_auth_oauthlib.flow import Flow def create_flow(redirect_uri: Optional[str] = None) -> Flow: + """Returns a Google Flow from client configuration.""" return Flow.from_client_config( settings.GOOGLE_FLOW, scopes=[ From c056ee0f431c8d18f0662d65b832b4022b4432da Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 14 May 2022 22:42:26 +0300 Subject: [PATCH 228/234] cleanup --- .../apps/api/viewsets/open_data_viewset.py | 58 +++++++++++++------ .../management/commands/import_tools.py | 42 +++++++------- .../commands/pull_from_aggregate.py | 28 ++++++--- 3 files changed, 81 insertions(+), 47 deletions(-) diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index 846fee3db2..d8a4db5323 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -1,3 +1,7 @@ +# -*- coding=utf-8 -*- +""" +The /api/v1/open-data implementation. +""" import json import re from collections import OrderedDict @@ -8,6 +12,7 @@ from django.http import StreamingHttpResponse from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ + from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response @@ -20,21 +25,21 @@ from onadata.apps.logger.models.xform import XForm, question_types_to_exclude from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.libs.data import parse_int -from onadata.libs.utils.logger_tools import remove_metadata_fields from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import StandardPageNumberPagination from onadata.libs.serializers.data_serializer import TableauDataSerializer from onadata.libs.serializers.open_data_serializer import OpenDataSerializer -from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.common_tags import ( ATTACHMENTS, - NOTES, GEOLOCATION, MULTIPLE_SELECT_TYPE, - REPEAT_SELECT_TYPE, NA_REP, + NOTES, + REPEAT_SELECT_TYPE, ) +from onadata.libs.utils.common_tools import json_stream +from onadata.libs.utils.logger_tools import remove_metadata_fields BaseViewset = get_baseviewset_class() IGNORED_FIELD_TYPES = ["select one", "select multiple"] @@ -46,11 +51,14 @@ DEFAULT_NA_REP = getattr(settings, "NA_REP", NA_REP) +# pylint: disable=invalid-name def replace_special_characters_with_underscores(data): + """Replaces special characters with underscores.""" return [re.sub(r"\W", r"_", a) for a in data] -def process_tableau_data(data, xform): +# pylint: disable=too-many-locals,too-many-statements +def process_tableau_data(data, xform): # noqa C901 """ Streamlines the row header fields with the column header fields for the same form. @@ -59,7 +67,8 @@ def process_tableau_data(data, xform): def get_xpath(key, nested_key): val = nested_key.split("/") - nested_key_diff = val[len(key.split("/")) :] + start_index = key.split("/").__len__() + nested_key_diff = val[start_index:] xpaths = key + f"[{index}]/" + "/".join(nested_key_diff) return xpaths @@ -86,7 +95,7 @@ def get_updated_data_dict(key, value, data_dict): return data_dict - def get_ordered_repeat_value(key, item, index): + def get_ordered_repeat_value(key, item): """ Return Ordered Dict of repeats in the order in which they appear in the XForm. @@ -115,6 +124,7 @@ def get_ordered_repeat_value(key, item, index): return data result = [] + # pylint: disable=too-many-nested-blocks if data: headers = xform.get_headers() tableau_headers = remove_metadata_fields(headers) @@ -129,7 +139,7 @@ def get_ordered_repeat_value(key, item, index): ]: for index, item in enumerate(value, start=1): # order repeat according to xform order - item = get_ordered_repeat_value(key, item, index) + item = get_ordered_repeat_value(key, item) flat_dict.update(item) else: try: @@ -141,7 +151,7 @@ def get_ordered_repeat_value(key, item, index): gps_xpaths = DataDictionary.get_additional_geopoint_xpaths( key ) - gps_parts = dict([(xpath, None) for xpath in gps_xpaths]) + gps_parts = {xpath: None for xpath in gps_xpaths} if len(parts) == 4: gps_parts = dict(zip(gps_xpaths, parts)) flat_dict.update(gps_parts) @@ -154,7 +164,10 @@ def get_ordered_repeat_value(key, item, index): return result +# pylint: disable=too-many-ancestors class OpenDataViewSet(ETagsMixin, CacheControlMixin, BaseViewset, ModelViewSet): + """The /api/v1/open-data API endpoint.""" + permission_classes = (OpenDataViewSetPermissions,) queryset = OpenData.objects.filter() lookup_field = "uuid" @@ -163,6 +176,7 @@ class OpenDataViewSet(ETagsMixin, CacheControlMixin, BaseViewset, ModelViewSet): MAX_INSTANCES_PER_REQUEST = 1000 pagination_class = StandardPageNumberPagination + # pylint: disable=no-self-use def get_tableau_type(self, xform_type): """ Returns a tableau-supported type based on a xform type. @@ -193,7 +207,7 @@ def get_tableau_column_headers(self): """ tableau_colulmn_headers = [] - def append_to_tableau_colulmn_headers(header, question_type=None): + def _append_to_tableau_colulmn_headers(header, question_type=None): quest_type = "string" if question_type: quest_type = question_type @@ -211,14 +225,14 @@ def append_to_tableau_colulmn_headers(header, question_type=None): # tableau. for header in xform_headers: for quest_name, quest_type in self.flattened_dict.items(): - if header == quest_name or header.endswith("_%s" % quest_name): - append_to_tableau_colulmn_headers(header, quest_type) + if header == quest_name or header.endswith(f"_{quest_name}"): + _append_to_tableau_colulmn_headers(header, quest_type) break else: if header == "_id": - append_to_tableau_colulmn_headers(header, "int") + _append_to_tableau_colulmn_headers(header, "int") else: - append_to_tableau_colulmn_headers(header) + _append_to_tableau_colulmn_headers(header) return tableau_colulmn_headers @@ -227,6 +241,7 @@ def data(self, request, **kwargs): """ Streams submission data response matching uuid in the request. """ + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() # get greater than value and cast it to an int gt_id = request.query_params.get("gt_id") @@ -237,7 +252,7 @@ def data(self, request, **kwargs): self.paginator.page_size_query_param, ] query_param_keys = request.query_params - should_paginate = any([k in query_param_keys for k in pagination_keys]) + should_paginate = any(k in query_param_keys for k in pagination_keys) data = [] if isinstance(self.object.content_object, XForm): @@ -275,12 +290,14 @@ def data(self, request, **kwargs): return Response(data) + # pylint: disable=no-self-use def get_streaming_response(self, data): """Get a StreamingHttpResponse response object""" def get_json_string(item): return json.dumps({re.sub(r"\W", r"_", a): b for a, b in item.items()}) + # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(data, get_json_string), content_type="application/json" ) @@ -292,11 +309,14 @@ def get_json_string(item): return response def destroy(self, request, *args, **kwargs): + """Deletes an OpenData object.""" self.get_object().delete() return Response(status=status.HTTP_204_NO_CONTENT) @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): + """Tableau schema - headers and table alias.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if isinstance(self.object.content_object, XForm): xform = self.object.content_object @@ -312,7 +332,7 @@ def schema(self, request, **kwargs): data = { "column_headers": tableau_column_headers, - "connection_name": "%s_%s" % (xform.project_id, xform.id_string), + "connection_name": f"{xform.project_id}_{xform.id_string}", "table_alias": xform.title, } @@ -320,8 +340,10 @@ def schema(self, request, **kwargs): return Response(status=status.HTTP_404_NOT_FOUND) + # pylint: disable=no-self-use @action(methods=["GET"], detail=False) def uuid(self, request, *args, **kwargs): + """Respond with the OpenData uuid.""" data_type = request.query_params.get("data_type") object_id = request.query_params.get("object_id") @@ -334,9 +356,9 @@ def uuid(self, request, *args, **kwargs): if data_type == "xform": xform = get_object_or_404(XForm, id=object_id) if request.user.has_perm("change_xform", xform): - ct = ContentType.objects.get_for_model(xform) + content_type = ContentType.objects.get_for_model(xform) _open_data = get_object_or_404( - OpenData, object_id=object_id, content_type=ct + OpenData, object_id=object_id, content_type=content_type ) if _open_data: return Response( diff --git a/onadata/apps/logger/management/commands/import_tools.py b/onadata/apps/logger/management/commands/import_tools.py index d01e3e1968..fbb8eb7d35 100644 --- a/onadata/apps/logger/management/commands/import_tools.py +++ b/onadata/apps/logger/management/commands/import_tools.py @@ -1,49 +1,49 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 coding=utf-8 - +# vim: ai ts=4 sts=4 et sw=4 +# -*- coding=utf-8 -*- +""" +import_tools - import ODK formms and instances. +""" import glob import os from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import gettext as _, gettext_lazy - -from onadata.libs.logger.import_tools import import_instances_from_zip -from onadata.libs.logger.models import Instance +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy +from onadata.apps.logger.import_tools import import_instances_from_zip +from onadata.apps.logger.models import Instance IMAGES_DIR = os.path.join(settings.MEDIA_ROOT, "attachments") class Command(BaseCommand): + """Import ODK forms and instances.""" + help = gettext_lazy("Import ODK forms and instances.") def handle(self, *args, **kwargs): + """Import ODK forms and instances.""" if args.__len__() < 2: raise CommandError(_("path(xform instances) username")) path = args[0] username = args[1] try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("Invalid username %s") % username) + user = get_user_model().objects.get(username=username) + except get_user_model().DoesNotExist as e: + raise CommandError(_(f"Invalid username {username}")) from e debug = False if debug: - self.stdout.write( - _("[Importing XForm Instances from %(path)s]\n") % {"path": path} - ) + self.stdout.write(_(f"[Importing XForm Instances from {path}]\n")) im_count = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) self.stdout.write(_("Before Parse:")) - self.stdout.write(_(" --> Images: %(nb)d") % {"nb": im_count}) - self.stdout.write( - (_(" --> Instances: %(nb)d") % {"nb": Instance.objects.count()}) - ) + self.stdout.write(_(f" --> Images: {im_count}")) + self.stdout.write((_(f" --> Instances: {Instance.objects.count()}"))) import_instances_from_zip(path, user) if debug: im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, "*"))) self.stdout.write(_("After Parse:")) - self.stdout.write(_(" --> Images: %(nb)d") % {"nb": im_count2}) - self.stdout.write( - (_(" --> Instances: %(nb)d") % {"nb": Instance.objects.count()}) - ) + self.stdout.write(_(f" --> Images: {im_count2}")) + self.stdout.write((_(f" --> Instances: {Instance.objects.count()}"))) diff --git a/onadata/apps/logger/management/commands/pull_from_aggregate.py b/onadata/apps/logger/management/commands/pull_from_aggregate.py index 88217bc7e6..07b7f75384 100644 --- a/onadata/apps/logger/management/commands/pull_from_aggregate.py +++ b/onadata/apps/logger/management/commands/pull_from_aggregate.py @@ -1,6 +1,13 @@ #!/usr/bin/env python +# -*- coding=utf-8 -*- +""" +pull_from_aggregate command -from django.contrib.auth.models import User +Uses the BriefcaseClient to download forms and submissions +from a server that implements the Briefcase Aggregate API. +""" + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ @@ -8,9 +15,14 @@ class Command(BaseCommand): - help = _("Insert all existing parsed instances into MongoDB") + """Download forms and submissions from a server with Briefcase Aggregate API""" + + help = _( + "Download forms and submissions from a server with Briefcase Aggregate API" + ) def add_arguments(self, parser): + """Download forms and submissions from a server with Briefcase Aggregate API""" parser.add_argument("--url", help=_("server url to pull forms and submissions")) parser.add_argument("-u", "--username", help=_("Username")) parser.add_argument("-p", "--password", help=_("Password")) @@ -20,15 +32,15 @@ def handle(self, *args, **kwargs): url = kwargs.get("url") username = kwargs.get("username") password = kwargs.get("password") - to = kwargs.get("to") - if username is None or password is None or to is None or url is None: + to_username = kwargs.get("to") + if username is None or password is None or to_username is None or url is None: self.stderr.write( - "pull_form_aggregate -u username -p password --to=username" + "pull_from_aggregate -u username -p password --to=username" " --url=aggregate_server_url" ) else: - user = User.objects.get(username=to) - bc = BriefcaseClient( + user = get_user_model().objects.get(username=to_username) + briefcase_client = BriefcaseClient( username=username, password=password, user=user, url=url ) - bc.download_xforms(include_instances=True) + briefcase_client.download_xforms(include_instances=True) From 89503b1fc2d243773bb648ea109e2dfa5cdc67d1 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Sat, 14 May 2022 23:08:57 +0300 Subject: [PATCH 229/234] cleanup --- .../apps/api/viewsets/open_data_viewset.py | 18 ++++++------- .../management/commands/update_xform_uuids.py | 27 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index d8a4db5323..b634d50909 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -62,17 +62,17 @@ def process_tableau_data(data, xform): # noqa C901 """ Streamlines the row header fields with the column header fields for the same form. - Handles Flattenning repeat data for tableau + Handles Flattening repeat data for tableau """ - def get_xpath(key, nested_key): + def get_xpath(key, nested_key, index): val = nested_key.split("/") start_index = key.split("/").__len__() nested_key_diff = val[start_index:] xpaths = key + f"[{index}]/" + "/".join(nested_key_diff) return xpaths - def get_updated_data_dict(key, value, data_dict): + def get_updated_data_dict(key, value, data_dict, index=0): """ Generates key, value pairs for select multiple question types. Defining the new xpaths from the @@ -88,14 +88,14 @@ def get_updated_data_dict(key, value, data_dict): try: for item in value: for (nested_key, nested_val) in item.items(): - xpath = get_xpath(key, nested_key) + xpath = get_xpath(key, nested_key, index) data_dict[xpath] = nested_val except AttributeError: data_dict[key] = value return data_dict - def get_ordered_repeat_value(key, item): + def get_ordered_repeat_value(key, item, index): """ Return Ordered Dict of repeats in the order in which they appear in the XForm. @@ -114,11 +114,11 @@ def get_ordered_repeat_value(key, item): # generate ["children", index, "immunization/polio_1"] for (nested_key, nested_val) in item_list.items(): qstn_type = xform.get_element(nested_key).type - xpaths = get_xpath(key, nested_key) + xpaths = get_xpath(key, nested_key, index) if qstn_type == MULTIPLE_SELECT_TYPE: - data = get_updated_data_dict(xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data, index) elif qstn_type == REPEAT_SELECT_TYPE: - data = get_updated_data_dict(xpaths, nested_val, data) + data = get_updated_data_dict(xpaths, nested_val, data, index) else: data[xpaths] = nested_val return data @@ -139,7 +139,7 @@ def get_ordered_repeat_value(key, item): ]: for index, item in enumerate(value, start=1): # order repeat according to xform order - item = get_ordered_repeat_value(key, item) + item = get_ordered_repeat_value(key, item, index) flat_dict.update(item) else: try: diff --git a/onadata/apps/logger/management/commands/update_xform_uuids.py b/onadata/apps/logger/management/commands/update_xform_uuids.py index 42840a487b..ecc0ed9c8c 100644 --- a/onadata/apps/logger/management/commands/update_xform_uuids.py +++ b/onadata/apps/logger/management/commands/update_xform_uuids.py @@ -1,5 +1,9 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +# -*- coding=utf-8 -*- +""" +update_xform_uuids command - Set uuid from a CSV file +""" import csv @@ -14,20 +18,23 @@ class Command(BaseCommand): + """Use a csv file with username, id_string and new_uuid to set new uuids.""" + help = gettext_lazy( - "Use a csv file with username, id_string and new_uuid to set new" " uuids" + "Use a csv file with username, id_string and new_uuid to set new uuids" ) def add_arguments(self, parser): parser.add_argument("-f", "--file", help=gettext_lazy("Path to csv file")) def handle(self, *args, **kwargs): + """Use a csv file with username, id_string and new_uuid to set new uuids.""" # all options are required if not kwargs.get("file"): raise CommandError("You must provide a path to the csv file") # try open the file try: - with open(kwargs.get("file"), "r") as f: + with open(kwargs.get("file"), "r", encoding="utf-8") as f: lines = csv.reader(f) i = 0 for line in lines: @@ -37,18 +44,16 @@ def handle(self, *args, **kwargs): uuid = line[2] update_xform_uuid(username, id_string, uuid) except IndexError: - self.stderr.write("line %d is in an invalid format" % (i + 1)) + self.stderr.write(f"line {i + 1} is in an invalid format") except XForm.DoesNotExist: self.stderr.write( - "XForm with username: %s and id " - "string: %s does not exist" % (username, id_string) + f"XForm with username: {username} and id " + f"string: {id_string} does not exist" ) except DuplicateUUIDError: - self.stderr.write( - "An xform with uuid: %s already exists" % uuid - ) + self.stderr.write(f"An xform with uuid: {uuid} already exists") else: i += 1 - self.stdout.write("Updated %d rows" % i) - except IOError: - raise CommandError("file %s could not be open" % kwargs.get("file")) + self.stdout.write(f"Updated {i} rows") + except IOError as e: + raise CommandError(f"file {kwargs.get('file')} could not be open") from e From 1cdc204963354ad1e292d4a0ecaba48a246f0d9a Mon Sep 17 00:00:00 2001 From: FrankApiyo Date: Tue, 17 May 2022 12:08:20 +0300 Subject: [PATCH 230/234] Allow geojson file uploads --- .../libs/serializers/metadata_serializer.py | 2 ++ .../tests/serializers/fixtures/sample.geojson | 13 ++++++++ .../serializers/test_metadata_serializer.py | 30 +++++++++++++++++++ onadata/settings/common.py | 1 + 4 files changed, 46 insertions(+) create mode 100644 onadata/libs/tests/serializers/fixtures/sample.geojson diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 983487c59a..bf92e52a1a 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -173,6 +173,8 @@ def validate(self, attrs): if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES + # add geojson mimetype + mimetypes.add_type('application/geo+json', '.geojson') data_content_type = ( data_file.content_type if data_file.content_type in allowed_types diff --git a/onadata/libs/tests/serializers/fixtures/sample.geojson b/onadata/libs/tests/serializers/fixtures/sample.geojson new file mode 100644 index 0000000000..1ae1b9cfdb --- /dev/null +++ b/onadata/libs/tests/serializers/fixtures/sample.geojson @@ -0,0 +1,13 @@ +{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 125.6, + 10.1 + ] + }, + "properties": { + "name": "Dinagat Islands" + } +} \ No newline at end of file diff --git a/onadata/libs/tests/serializers/test_metadata_serializer.py b/onadata/libs/tests/serializers/test_metadata_serializer.py index 79da930241..7c92b5697f 100644 --- a/onadata/libs/tests/serializers/test_metadata_serializer.py +++ b/onadata/libs/tests/serializers/test_metadata_serializer.py @@ -97,3 +97,33 @@ def test_svg_media_files(self): self.assertEqual( serializer.validated_data["data_file_type"], "image/svg+xml" ) + + def test_geojson_media_files(self): + """ + Test that an geojson file is uploaded ok + """ + self._login_user_and_profile() + self._publish_form_with_hxl_support() + data_value = 'sample.geojson' + path = os.path.join(os.path.dirname(__file__), 'fixtures', + 'sample.geojson') + with open(path) as f: + f = InMemoryUploadedFile( + f, 'media', data_value, None, 2324, None) + data = { + 'data_value': data_value, + 'data_file': f, + 'data_type': 'media', + 'xform': self.xform.pk + } + serializer = MetaDataSerializer(data=data) + self.assertTrue(serializer.is_valid()) + self.assertEqual( + serializer.validated_data['data_file_type'], + 'application/geo+json') + self.assertEqual( + serializer.validated_data['data_value'], + 'sample.geojson') + self.assertEqual( + serializer.validated_data['data_type'], + 'media') diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 43665539a2..b4c5892189 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -512,6 +512,7 @@ def configure_logging(logger, **kwargs): "video/3gpp", "video/mp4", "application/json", + "application/geo+json", "application/pdf", "application/msword", "application/vnd.ms-excel", From 0339ebc1cb4671884aae1684ac4e2d016246a4b4 Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 18 May 2022 11:52:17 +0300 Subject: [PATCH 231/234] Bump version to 3.0.0 Signed-off-by: Kipchirchir Sigei --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index fb2e3da22c..078bcd8c5a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [metadata] name = onadata +version = 3.0.0 description = Collect Analyze and Share Data long_description = file: README.rst long_description_content_type = text/x-rst From ac2110606d8ff7414ca1d2795670bea7e03cb35d Mon Sep 17 00:00:00 2001 From: Kipchirchir Sigei Date: Wed, 18 May 2022 11:52:51 +0300 Subject: [PATCH 232/234] Move test deps to tests_require Signed-off-by: Kipchirchir Sigei --- setup.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 078bcd8c5a..9b9a5265e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,11 @@ project_urls = [options] packages = find: platforms = any +tests_require = + flake8 + mock + httmock + requests-mock install_requires = Django>=3.2.13,<4 django-guardian @@ -65,9 +70,6 @@ install_requires = pyxform #spss savreaderwriter - #tests - mock - httmock #memcached support pylibmc python-memcached @@ -89,10 +91,8 @@ install_requires = python-dateutil pytz requests - requests-mock simplejson uwsgi - flake8 raven django-activity-stream paho-mqtt From 75fae728a50e79b62b6642374ee4dc0543fa79dc Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 19 May 2022 21:07:41 +0300 Subject: [PATCH 233/234] Compare with lowercase export_type --- onadata/apps/viewer/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 76ca8fc6b9..7373fb23ea 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -483,7 +483,7 @@ def export_list(request, username, id_string, export_type): # noqa C901 """ credential = None - if export_type not in Export.EXPORT_TYPE_DICT: + if export_type.lower() not in Export.EXPORT_TYPE_DICT: return HttpResponseBadRequest( _(f'Export type "{export_type}" is not supported.') ) From fdb0fcb2d311122f71c7dbf39151385a8c395661 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 19 May 2022 21:09:39 +0300 Subject: [PATCH 234/234] Set version in __init__.py --- onadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/__init__.py b/onadata/__init__.py index e6f56e01e0..8616e0207a 100644 --- a/onadata/__init__.py +++ b/onadata/__init__.py @@ -6,7 +6,7 @@ """ from __future__ import absolute_import, unicode_literals -__version__ = "2.5.20" +__version__ = "3.0.0" # This will make sure the app is always imported when