From 5bc6ac135dd9c18a1c86b96e6f783295a9e55421 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 8 Oct 2020 14:37:51 +0300 Subject: [PATCH 1/6] Update base, dev, s3 & ses requirements --- requirements/base.pip | 167 ++++++++++++++++++------------------ requirements/dev.pip | 193 +++++++++++++++++++++--------------------- requirements/s3.in | 1 + requirements/s3.pip | 23 +++-- requirements/ses.in | 1 + requirements/ses.pip | 19 +++-- setup.py | 6 +- 7 files changed, 211 insertions(+), 199 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index e3215c007a..97da65566c 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -11,134 +11,137 @@ -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest # via -r requirements/base.in -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient # via -r requirements/base.in alabaster==0.7.12 # via sphinx -amqp==2.5.2 # via kombu +amqp==5.0.1 # via kombu analytics-python==1.2.9 # via onadata appoptics-metrics==5.1.0 # via onadata argparse==1.4.0 # via unittest2 -attrs==19.3.0 # via jsonschema +attrs==20.2.0 # via jsonschema babel==2.8.0 # via sphinx -billiard==3.6.2.0 # via celery -boto3==1.12.4 # via tabulator -botocore==1.15.4 # via boto3, s3transfer -cchardet==2.1.5 # via datapackage, tabulator -celery==4.4.0 # via django-celery-results, onadata -certifi==2019.11.28 # via requests -cffi==1.14.0 # via cryptography -chardet==3.0.4 # via requests -click==7.0 # via datapackage, tableschema, tabulator -cryptography==2.8 # via onadata -datapackage==1.11.1 # via pyfloip +billiard==3.6.3.0 # via celery +boto3==1.15.17 # via tabulator +botocore==1.18.17 # via boto3, s3transfer +cached-property==1.5.2 # via tableschema +celery==5.0.0 # via onadata +certifi==2020.6.20 # via requests +cffi==1.14.3 # via cryptography +chardet==3.0.4 # via datapackage, requests, tabulator +click-didyoumean==0.0.3 # via celery +click-repl==0.1.6 # via celery +click==7.1.2 # via celery, click-didyoumean, click-repl, datapackage, tableschema, tabulator +cryptography==3.1.1 # via onadata +datapackage==1.15.1 # via pyfloip defusedxml==0.6.0 # via djangorestframework-xml -dict2xml==1.6.1 # via onadata -django-activity-stream==0.8.0 # via onadata -django-celery-results==1.2.0 # via onadata -django-cors-headers==3.2.1 # via onadata -django-debug-toolbar==2.2 # via onadata -django-filter==2.2.0 # via onadata -django-guardian==2.1.0 # via djangorestframework-guardian, onadata -django-nose==1.4.6 # via onadata -django-oauth-toolkit==1.2.0 # via onadata -django-ordered-model==3.3.0 # via onadata +dict2xml==1.7.0 # via onadata +django-activity-stream==0.9.0 # via onadata +django-cors-headers==3.5.0 # via onadata +django-debug-toolbar==3.1.1 # via onadata +django-filter==2.4.0 # via onadata +django-guardian==2.3.0 # via djangorestframework-guardian, onadata +django-nose==1.4.7 # via onadata +django-oauth-toolkit==1.3.2 # via onadata +django-ordered-model==3.4.1 # via onadata django-query-builder==1.2.0 # via onadata -django-registration-redux==2.7 # via onadata -django-render-block==0.6 # via django-templated-email -django-reversion==3.0.7 # via onadata -django-taggit==1.2.0 # via onadata +django-registration-redux==2.8 # via onadata +django-render-block==0.8.1 # via django-templated-email +django-reversion==3.0.8 # via onadata +django-taggit==1.3.0 # via onadata django-templated-email==2.3.0 # via onadata -django==2.2.10 # via django-cors-headers, django-debug-toolbar, django-filter, django-oauth-toolkit, django-query-builder, django-render-block, django-reversion, django-taggit, djangorestframework, djangorestframework-guardian, djangorestframework-jsonapi, jsonfield, onadata +django==2.2.16 # via django-cors-headers, django-debug-toolbar, django-filter, django-guardian, django-oauth-toolkit, django-query-builder, django-render-block, django-reversion, django-taggit, djangorestframework, djangorestframework-guardian, djangorestframework-jsonapi, jsonfield, onadata djangorestframework-csv==2.1.0 # via onadata -djangorestframework-gis==0.15 # via onadata +djangorestframework-gis==0.16 # via onadata djangorestframework-guardian==0.3.0 # via onadata -djangorestframework-jsonapi==3.1.0 # via onadata +djangorestframework-jsonapi==3.2.0 # via onadata djangorestframework-jsonp==1.0.2 # via onadata -djangorestframework-xml==1.4.0 # via onadata -djangorestframework==3.11.0 # via djangorestframework-csv, djangorestframework-gis, djangorestframework-guardian, djangorestframework-jsonapi, onadata -docutils==0.15.2 # via botocore, sphinx +djangorestframework-xml==2.0.0 # via onadata +djangorestframework==3.11.2 # via djangorestframework-csv, djangorestframework-gis, djangorestframework-guardian, djangorestframework-jsonapi, onadata +docutils==0.16 # via sphinx dpath==2.0.1 # via onadata elaphe3==0.2.0 # via onadata -entrypoints==0.3 # via flake8 et-xmlfile==1.0.1 # via openpyxl -flake8==3.7.9 # via onadata +flake8==3.8.4 # via onadata fleming==0.5.0 # via django-query-builder -formencode==1.3.1 # via pyxform +formencode==2.0.0 # via pyxform future==0.18.2 # via python-json2xlsclient geojson==2.5.0 # via onadata httmock==1.3.0 # via onadata -httplib2==0.17.0 # via oauth2client, onadata -idna==2.9 # via requests -ijson==2.6.1 # via tabulator +httplib2==0.18.1 # via oauth2client, onadata +idna==2.10 # via requests +ijson==3.1.2 # via tabulator imagesize==1.2.0 # via sphinx -importlib-metadata==1.5.0 # via jsonschema, kombu -inflection==0.3.1 # via djangorestframework-jsonapi +importlib-metadata==2.0.0 # via flake8, jsonpickle, jsonschema, kombu, markdown +inflection==0.5.1 # via djangorestframework-jsonapi isodate==0.6.0 # via tableschema jdcal==1.4.1 # via openpyxl -jinja2==2.11.1 # via sphinx -jmespath==0.9.4 # via boto3, botocore +jinja2==2.11.2 # via sphinx +jmespath==0.10.0 # via boto3, botocore jsonfield==0.9.23 # via onadata jsonlines==1.2.0 # via tabulator -jsonpickle==1.3 # via onadata +jsonpickle==1.4.1 # via onadata jsonpointer==2.0 # via datapackage jsonschema==3.2.0 # via datapackage, tableschema -kombu==4.6.7 # via celery +kombu==5.0.2 # via celery linear-tsv==1.1.0 # via tabulator linecache2==1.0.0 # via traceback2 -lxml==4.5.0 # via onadata -markdown==3.2.1 # via onadata +lxml==4.5.2 # via onadata +markdown==3.3.1 # via onadata markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via flake8 -mock==4.0.1 # via onadata +mock==4.0.2 # via onadata modilabs-python-utils==0.1.5 # via onadata nose==1.3.7 # via django-nose -numpy==1.18.1 # via onadata +numpy==1.19.2 # via onadata oauthlib==3.1.0 # via django-oauth-toolkit -openpyxl==3.0.3 # via onadata, tabulator -packaging==20.1 # via sphinx -paho-mqtt==1.5.0 # via onadata -pillow==7.0.0 # via elaphe3, onadata -psycopg2==2.8.4 # via onadata +openpyxl==3.0.5 # via onadata, tabulator +packaging==20.4 # via sphinx +paho-mqtt==1.5.1 # via onadata +pillow==8.0.0 # via elaphe3, onadata +prompt-toolkit==3.0.8 # via click-repl +psycopg2==2.8.6 # via onadata pyasn1-modules==0.2.8 # via oauth2client pyasn1==0.4.8 # via oauth2client, pyasn1-modules, rsa -pycodestyle==2.5.0 # via flake8 -pycparser==2.19 # via cffi -pyflakes==2.1.1 # via flake8 -pygments==2.5.2 # via sphinx +pycodestyle==2.6.0 # via flake8 +pycparser==2.20 # via cffi +pyflakes==2.2.0 # via flake8 +pygments==2.7.1 # via sphinx pyjwt==1.7.1 # via onadata pylibmc==1.6.1 # via onadata -pymongo==3.10.1 # via onadata -pyparsing==2.4.6 # via packaging -pyrsistent==0.15.7 # via jsonschema +pymongo==3.11.0 # via onadata +pyparsing==2.4.7 # via packaging +pyrsistent==0.17.3 # via jsonschema python-dateutil==2.8.1 # via analytics-python, botocore, fleming, onadata, tableschema python-memcached==1.59 # via onadata -pytz==2019.3 # via babel, celery, django, django-query-builder, fleming, onadata +pytz==2020.1 # via babel, celery, django, django-query-builder, fleming, onadata pyxform==1.1.0 # via onadata, pyfloip raven==6.10.0 # via onadata recaptcha-client==1.0.6 # via onadata -requests-mock==1.7.0 # via onadata -requests==2.23.0 # via analytics-python, datapackage, django-oauth-toolkit, httmock, onadata, python-json2xlsclient, requests-mock, sphinx, tableschema, tabulator -rfc3986==1.3.2 # via tableschema -rsa==4.0 # via oauth2client +requests-mock==1.8.0 # via onadata +requests==2.24.0 # via analytics-python, datapackage, django-oauth-toolkit, httmock, onadata, python-json2xlsclient, requests-mock, sphinx, tableschema, tabulator +rfc3986==1.4.0 # via tableschema +rsa==4.6 # via oauth2client s3transfer==0.3.3 # via boto3 savreaderwriter==3.4.2 # via onadata -simplejson==3.17.0 # via onadata -six==1.14.0 # via analytics-python, appoptics-metrics, cryptography, datapackage, dict2xml, django-query-builder, django-templated-email, djangorestframework-csv, isodate, jsonlines, jsonschema, linear-tsv, oauth2client, packaging, pyrsistent, python-dateutil, python-memcached, requests-mock, tableschema, tabulator, unittest2 +simplejson==3.17.2 # via onadata +six==1.15.0 # via analytics-python, appoptics-metrics, click-repl, cryptography, datapackage, django-query-builder, django-templated-email, djangorestframework-csv, formencode, isodate, jsonlines, jsonschema, linear-tsv, oauth2client, packaging, python-dateutil, python-memcached, requests-mock, tableschema, tabulator, unittest2 snowballstemmer==2.0.0 # via sphinx -sphinx==2.4.2 # via onadata -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx +sphinx==3.2.1 # via onadata +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy==1.3.13 # via tabulator -sqlparse==0.3.0 # via django, django-debug-toolbar -tableschema==1.13.1 # via datapackage -tabulator==1.35.0 # via datapackage, tableschema +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy==1.3.20 # via tabulator +sqlparse==0.4.1 # via django, django-debug-toolbar +tableschema==1.20.0 # via datapackage +tabulator==1.52.4 # via datapackage, tableschema traceback2==1.4.0 # via unittest2 unicodecsv==0.14.1 # via datapackage, djangorestframework-csv, onadata, pyxform, tableschema, tabulator unittest2==1.1.0 # via pyxform -urllib3==1.25.8 # via botocore, requests -uwsgi==2.0.18 # via onadata -vine==1.3.0 # via amqp, celery +urllib3==1.25.10 # via botocore, requests +uwsgi==2.0.19.1 # via onadata +vine==5.0.0 # via amqp, celery +wcwidth==0.2.5 # via prompt-toolkit xlrd==1.2.0 # via onadata, pyxform, tabulator xlwt==1.3.0 # via onadata -zipp==3.0.0 # via importlib-metadata +zipp==3.3.1 # via importlib-metadata + diff --git a/requirements/dev.pip b/requirements/dev.pip index e7c72e29e7..dfa71e8874 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -11,157 +11,158 @@ -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest # via -r requirements/base.in -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient # via -r requirements/base.in alabaster==0.7.12 # via sphinx -amqp==2.5.2 # via kombu +amqp==5.0.1 # via kombu analytics-python==1.2.9 # via onadata appnope==0.1.0 # via ipython appoptics-metrics==5.1.0 # via onadata argparse==1.4.0 # via unittest2 astroid==1.6.6 # via pylint -attrs==19.3.0 # via jsonschema +attrs==20.2.0 # via jsonschema babel==2.8.0 # via sphinx -backcall==0.1.0 # via ipython -billiard==3.6.2.0 # via celery -boto3==1.12.4 # via tabulator -botocore==1.15.4 # via boto3, s3transfer -cchardet==2.1.5 # via datapackage, tabulator -celery==4.4.0 # via django-celery-results, onadata -certifi==2019.11.28 # via requests -cffi==1.14.0 # via cryptography -chardet==3.0.4 # via requests -click==7.0 # via datapackage, tableschema, tabulator -cryptography==2.8 # via onadata -datapackage==1.11.1 # via pyfloip -decorator==4.4.1 # via ipython, traitlets +backcall==0.2.0 # via ipython +billiard==3.6.3.0 # via celery +boto3==1.15.17 # via tabulator +botocore==1.18.17 # via boto3, s3transfer +cached-property==1.5.2 # via tableschema +celery==5.0.0 # via onadata +certifi==2020.6.20 # via requests +cffi==1.14.3 # via cryptography +chardet==3.0.4 # via datapackage, requests, tabulator +click-didyoumean==0.0.3 # via celery +click-repl==0.1.6 # via celery +click==7.1.2 # via celery, click-didyoumean, click-repl, datapackage, tableschema, tabulator +cryptography==3.1.1 # via onadata +datapackage==1.15.1 # via pyfloip +decorator==4.4.2 # via ipython defusedxml==0.6.0 # via djangorestframework-xml -dict2xml==1.6.1 # via onadata -django-activity-stream==0.8.0 # via onadata -django-celery-results==1.2.0 # via onadata -django-cors-headers==3.2.1 # via onadata -django-debug-toolbar==2.2 # via onadata -django-extensions==2.2.8 # via -r requirements/dev.in -django-filter==2.2.0 # via onadata -django-guardian==2.2.0 # via djangorestframework-guardian, onadata -django-nose==1.4.6 # via onadata -django-oauth-toolkit==1.2.0 # via onadata -django-ordered-model==3.3.0 # via onadata +dict2xml==1.7.0 # via onadata +django-activity-stream==0.9.0 # via onadata +django-cors-headers==3.5.0 # via onadata +django-debug-toolbar==3.1.1 # via onadata +django-extensions==3.0.9 # via -r requirements/dev.in +django-filter==2.4.0 # via onadata +django-guardian==2.3.0 # via djangorestframework-guardian, onadata +django-nose==1.4.7 # via onadata +django-oauth-toolkit==1.3.2 # via onadata +django-ordered-model==3.4.1 # via onadata django-query-builder==1.2.0 # via onadata -django-registration-redux==2.7 # via onadata -django-render-block==0.6 # via django-templated-email -django-reversion==3.0.7 # via onadata -django-taggit==1.2.0 # via onadata +django-registration-redux==2.8 # via onadata +django-render-block==0.8.1 # via django-templated-email +django-reversion==3.0.8 # via onadata +django-taggit==1.3.0 # via onadata django-templated-email==2.3.0 # via onadata -django==2.2.10 # via django-cors-headers, django-debug-toolbar, django-filter, django-guardian, django-oauth-toolkit, django-query-builder, django-render-block, django-reversion, django-taggit, djangorestframework, djangorestframework-guardian, djangorestframework-jsonapi, jsonfield, onadata +django==2.2.16 # via django-cors-headers, django-debug-toolbar, django-filter, django-guardian, django-oauth-toolkit, django-query-builder, django-render-block, django-reversion, django-taggit, djangorestframework, djangorestframework-guardian, djangorestframework-jsonapi, jsonfield, onadata djangorestframework-csv==2.1.0 # via onadata -djangorestframework-gis==0.15 # via onadata +djangorestframework-gis==0.16 # via onadata djangorestframework-guardian==0.3.0 # via onadata -djangorestframework-jsonapi==3.1.0 # via onadata +djangorestframework-jsonapi==3.2.0 # via onadata djangorestframework-jsonp==1.0.2 # via onadata -djangorestframework-xml==1.4.0 # via onadata -djangorestframework==3.11.0 # via djangorestframework-csv, djangorestframework-gis, djangorestframework-guardian, djangorestframework-jsonapi, onadata -docutils==0.15.2 # via botocore, sphinx +djangorestframework-xml==2.0.0 # via onadata +djangorestframework==3.11.2 # via djangorestframework-csv, djangorestframework-gis, djangorestframework-guardian, djangorestframework-jsonapi, onadata +docutils==0.16 # via sphinx dpath==2.0.1 # via onadata elaphe3==0.2.0 # via onadata -entrypoints==0.3 # via flake8 et-xmlfile==1.0.1 # via openpyxl -flake8==3.7.9 # via onadata +flake8==3.8.4 # via onadata fleming==0.5.0 # via django-query-builder -formencode==1.3.1 # via pyxform +formencode==2.0.0 # via pyxform future==0.18.2 # via python-json2xlsclient geojson==2.5.0 # via onadata httmock==1.3.0 # via onadata -httplib2==0.17.0 # via oauth2client, onadata -idna==2.9 # via requests -ijson==2.6.1 # via tabulator +httplib2==0.18.1 # via oauth2client, onadata +idna==2.10 # via requests +ijson==3.1.2 # via tabulator imagesize==1.2.0 # via sphinx -importlib-metadata==1.5.0 # via jsonschema, kombu -inflection==0.3.1 # via djangorestframework-jsonapi -ipdb==0.12.3 # via -r requirements/dev.in +importlib-metadata==2.0.0 # via flake8, jsonpickle, jsonschema, kombu, markdown +inflection==0.5.1 # via djangorestframework-jsonapi +ipdb==0.13.4 # via -r requirements/dev.in ipython-genutils==0.2.0 # via traitlets -ipython==7.12.0 # via ipdb +ipython==7.18.1 # via ipdb isodate==0.6.0 # via tableschema -isort==4.3.21 # via -r requirements/dev.in, pylint +isort==5.6.4 # via -r requirements/dev.in, pylint jdcal==1.4.1 # via openpyxl -jedi==0.16.0 # via ipython -jinja2==2.11.1 # via sphinx -jmespath==0.9.4 # via boto3, botocore +jedi==0.17.2 # via ipython +jinja2==2.11.2 # via sphinx +jmespath==0.10.0 # via boto3, botocore jsonfield==0.9.23 # via onadata jsonlines==1.2.0 # via tabulator -jsonpickle==1.3 # via onadata +jsonpickle==1.4.1 # via onadata jsonpointer==2.0 # via datapackage jsonschema==3.2.0 # via datapackage, tableschema -kombu==4.6.7 # via celery -lazy-object-proxy==1.4.3 # via astroid +kombu==5.0.2 # via celery +lazy-object-proxy==1.5.1 # via astroid linear-tsv==1.1.0 # via tabulator linecache2==1.0.0 # via traceback2 -lxml==4.5.0 # via onadata -markdown==3.2.1 # via onadata +lxml==4.5.2 # via onadata +markdown==3.3.1 # via onadata markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via flake8, pylint -mock==4.0.1 # via onadata +mock==4.0.2 # via onadata modilabs-python-utils==0.1.5 # via onadata nose==1.3.7 # via django-nose -numpy==1.18.1 # via onadata +numpy==1.19.2 # via onadata oauthlib==3.1.0 # via django-oauth-toolkit -openpyxl==3.0.3 # via onadata, tabulator -packaging==20.1 # via sphinx -paho-mqtt==1.5.0 # via onadata -parso==0.6.1 # via jedi +openpyxl==3.0.5 # via onadata, tabulator +packaging==20.4 # via sphinx +paho-mqtt==1.5.1 # via onadata +parso==0.7.1 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==7.0.0 # via elaphe3, onadata -prompt-toolkit==3.0.3 # via ipython -psycopg2==2.8.4 # via onadata +pillow==8.0.0 # via elaphe3, onadata +prompt-toolkit==3.0.8 # via click-repl, ipython +psycopg2==2.8.6 # via onadata ptyprocess==0.6.0 # via pexpect pyasn1-modules==0.2.8 # via oauth2client pyasn1==0.4.8 # via oauth2client, pyasn1-modules, rsa -pycodestyle==2.5.0 # via flake8 -pycparser==2.19 # via cffi -pyflakes==2.1.1 # via flake8 -pygments==2.5.2 # via ipython, sphinx +pycodestyle==2.6.0 # via flake8 +pycparser==2.20 # via cffi +pyflakes==2.2.0 # via flake8 +pygments==2.7.1 # via ipython, sphinx pyjwt==1.7.1 # via onadata pylibmc==1.6.1 # via onadata pylint-django==0.11.1 # via -r requirements/dev.in pylint-plugin-utils==0.6 # via pylint-django pylint==1.9.4 # via -r requirements/dev.in, pylint-django, pylint-plugin-utils -pymongo==3.10.1 # via onadata -pyparsing==2.4.6 # via packaging -pyrsistent==0.15.7 # via jsonschema +pymongo==3.11.0 # via onadata +pyparsing==2.4.7 # via packaging +pyrsistent==0.17.3 # via jsonschema python-dateutil==2.8.1 # via analytics-python, botocore, fleming, onadata, tableschema python-memcached==1.59 # via onadata -pytz==2019.3 # via babel, celery, django, django-query-builder, fleming, onadata +pytz==2020.1 # via babel, celery, django, django-query-builder, fleming, onadata pyxform==1.1.0 # via onadata, pyfloip raven==6.10.0 # via onadata recaptcha-client==1.0.6 # via onadata -requests-mock==1.7.0 # via onadata -requests==2.23.0 # via analytics-python, datapackage, django-oauth-toolkit, httmock, onadata, python-json2xlsclient, requests-mock, sphinx, tableschema, tabulator -rfc3986==1.3.2 # via tableschema -rsa==4.0 # via oauth2client +requests-mock==1.8.0 # via onadata +requests==2.24.0 # via analytics-python, datapackage, django-oauth-toolkit, httmock, onadata, python-json2xlsclient, requests-mock, sphinx, tableschema, tabulator +rfc3986==1.4.0 # via tableschema +rsa==4.6 # via oauth2client s3transfer==0.3.3 # via boto3 savreaderwriter==3.4.2 # via onadata -simplejson==3.17.0 # via onadata -six==1.14.0 # via analytics-python, appoptics-metrics, astroid, cryptography, datapackage, dict2xml, django-extensions, django-query-builder, django-templated-email, djangorestframework-csv, isodate, jsonlines, jsonschema, linear-tsv, oauth2client, packaging, pylint, pyrsistent, python-dateutil, python-memcached, requests-mock, tableschema, tabulator, traitlets, unittest2 +simplejson==3.17.2 # via onadata +six==1.15.0 # via analytics-python, appoptics-metrics, astroid, click-repl, cryptography, datapackage, django-query-builder, django-templated-email, djangorestframework-csv, formencode, isodate, jsonlines, jsonschema, linear-tsv, oauth2client, packaging, pylint, python-dateutil, python-memcached, requests-mock, tableschema, tabulator, unittest2 snowballstemmer==2.0.0 # via sphinx -sphinx==2.4.2 # via onadata -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx +sphinx==3.2.1 # via onadata +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy==1.3.13 # via tabulator -sqlparse==0.3.0 # via django, django-debug-toolbar -tableschema==1.13.1 # via datapackage -tabulator==1.35.0 # via datapackage, tableschema +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy==1.3.20 # via tabulator +sqlparse==0.4.1 # via django, django-debug-toolbar +tableschema==1.20.0 # via datapackage +tabulator==1.52.4 # via datapackage, tableschema traceback2==1.4.0 # via unittest2 -traitlets==4.3.3 # via ipython +traitlets==5.0.5 # via ipython unicodecsv==0.14.1 # via datapackage, djangorestframework-csv, onadata, pyxform, tableschema, tabulator unittest2==1.1.0 # via pyxform -urllib3==1.25.8 # via botocore, requests -uwsgi==2.0.18 # via onadata -vine==1.3.0 # via amqp, celery -wcwidth==0.1.8 # via prompt-toolkit -wrapt==1.12.0 # via astroid +urllib3==1.25.10 # via botocore, requests +uwsgi==2.0.19.1 # via onadata +vine==5.0.0 # via amqp, celery +wcwidth==0.2.5 # via prompt-toolkit +wrapt==1.12.1 # via astroid xlrd==1.2.0 # via onadata, pyxform, tabulator xlwt==1.3.0 # via onadata -yapf==0.29.0 # via -r requirements/dev.in -zipp==3.0.0 # via importlib-metadata +yapf==0.30.0 # via -r requirements/dev.in +zipp==3.3.1 # via importlib-metadata + diff --git a/requirements/s3.in b/requirements/s3.in index 0837287fda..88aa5acdeb 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,2 +1,3 @@ django-storages +django >=2.2,<3 boto3 diff --git a/requirements/s3.pip b/requirements/s3.pip index b4d5270057..dadbffc53a 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -4,15 +4,14 @@ # # pip-compile --output-file=requirements/s3.pip requirements/s3.in # -boto3==1.9.92 -botocore==1.12.92 # via boto3, s3transfer -django-storages==1.7.1 -django==2.2.9 # via django-storages -docutils==0.14 # via botocore -jmespath==0.9.3 # via boto3, botocore -python-dateutil==2.8.0 # via botocore -pytz==2018.9 # via django -s3transfer==0.2.0 # via boto3 -six==1.12.0 # via python-dateutil -sqlparse==0.3.0 # via django -urllib3==1.24.1 # via botocore +boto3==1.15.14 # via -r requirements/s3.in +botocore==1.18.14 # via boto3, s3transfer +django-storages==1.10.1 # via -r requirements/s3.in +django==2.2.16 # via -r requirements/s3.in, django-storages +jmespath==0.10.0 # via boto3, botocore +python-dateutil==2.8.1 # via botocore +pytz==2020.1 # via django +s3transfer==0.3.3 # via boto3 +six==1.15.0 # via python-dateutil +sqlparse==0.4.1 # via django +urllib3==1.25.10 # via botocore diff --git a/requirements/ses.in b/requirements/ses.in index 99466e6e89..e53db47c0e 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,2 +1,3 @@ boto +django >=2.2,<3 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index f6bb6e9909..9253b9e8b4 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -2,9 +2,18 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements/ses.pip requirements/ses.in +# pip-compile --output-file=requirements/ses.pip requirements/ses.in # -boto==2.49.0 -django-ses==0.8.9 -future==0.17.1 # via django-ses -pytz==2018.9 # via django-ses +boto3==1.15.14 # via django-ses +boto==2.49.0 # via -r requirements/ses.in +botocore==1.18.14 # via boto3, s3transfer +django-ses==1.0.3 # via -r requirements/ses.in +django==2.2.16 # via -r requirements/ses.in, django-ses +future==0.18.2 # via django-ses +jmespath==0.10.0 # via boto3, botocore +python-dateutil==2.8.1 # via botocore +pytz==2020.1 # via django, django-ses +s3transfer==0.3.3 # via boto3 +six==1.15.0 # via python-dateutil +sqlparse==0.4.1 # via django +urllib3==1.25.10 # via botocore diff --git a/setup.py b/setup.py index 401e81f4de..436ef7b7db 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ }, packages=find_packages(exclude=['docs', 'tests']), install_requires=[ - "Django>=2.2,<2.3", + "Django>=2.2,<3", "django-guardian", "django-registration-redux", "django-templated-email", @@ -42,7 +42,6 @@ # generic relation "django-query-builder", "celery", - "django-celery-results", # cors "django-cors-headers", "django-debug-toolbar", @@ -73,7 +72,7 @@ "dict2xml", "lxml", # pyxform - "pyxform", + "pyxform <= 1.1.0", # spss "savreaderwriter", # tests @@ -120,7 +119,6 @@ 'https://github.com/onaio/floip-py/tarball/3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=floip', # noqa pylint: disable=line-too-long 'https://github.com/onaio/python-json2xlsclient/tarball/62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient', # noqa pylint: disable=line-too-long 'https://github.com/onaio/oauth2client/tarball/75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client', # noqa pylint: disable=line-too-long - ], extras_require={ ':python_version=="2.7"': [ From c4b0354c8802a8a4cf8a05bc5abfe5b2ecae3362 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 8 Oct 2020 14:38:55 +0300 Subject: [PATCH 2/6] Remove unused variable --- onadata/apps/sms_support/tools.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index 4b71a9ff9d..f6af0471f8 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -172,9 +172,6 @@ def check_form_sms_compatibility(form, json_survey=None): if json_survey is None: json_survey = form.get('form_o', {}) - form_text = u"%s
" % form['text'] - else: - form_text = u"" def prep_return(msg, comp=None): @@ -208,7 +205,7 @@ def prep_return(msg, comp=None): return {'type': alert, 'text': u"%(msg)s%(outro)s" - % {'intro': form_text, 'msg': msg, 'outro': outro}} + % {'msg': msg, 'outro': outro}} # first level children. should be groups groups = json_survey.get('children', [{}]) From ab365975a4f24ff8384bda579b578b8a7c955412 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Fri, 25 Sep 2020 14:30:01 +0300 Subject: [PATCH 3/6] Silence linting errors - Remove ambiguous variable names - Correctly indent lines --- onadata/apps/main/models/audit.py | 8 +- onadata/apps/main/urls.py | 287 ++++++++++--------- onadata/apps/viewer/parsed_instance_tools.py | 4 +- onadata/libs/data/query.py | 4 +- onadata/libs/utils/csv_builder.py | 4 +- 5 files changed, 158 insertions(+), 149 deletions(-) diff --git a/onadata/apps/main/models/audit.py b/onadata/apps/main/models/audit.py index 84fb88a550..3a4d0ef4d2 100644 --- a/onadata/apps/main/models/audit.py +++ b/onadata/apps/main/models/audit.py @@ -74,9 +74,11 @@ def query_data(cls, username, query=None, fields=None, sort=None, start=0, or_params = [] if '$or' in list(query): or_dict = query.pop('$or') - for l in or_dict: - or_where.extend([u"json->>%s = %s" for i in l.items()]) - [or_params.extend(i) for i in l.items()] + for or_query in or_dict: + or_where.extend( + [u"json->>%s = %s" for i in or_query.items()]) + [ # pylint: disable=expression-not-assigned + or_params.extend(i) for i in or_query.items()] or_where = [u"".join([u"(", u" OR ".join(or_where), u")"])] diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index c812341947..eda7723424 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -41,9 +41,9 @@ re_path(r'^i18n/', include(i18n)), url('^api/v1/', include(router.urls)), re_path(r'^api-docs/', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), + RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), re_path(r'^api/$', - RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), + RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), re_path(r'^api/v1$', RedirectView.as_view(url='/api/v1/', permanent=True)), # django default stuff @@ -53,16 +53,16 @@ # oath2_provider re_path(r'^o/authorize/$', main_views.OnaAuthorizationView.as_view(), - name="oauth2_provider_authorize"), + name="oauth2_provider_authorize"), re_path(r'^o/', include('oauth2_provider.urls', - namespace='oauth2_provider')), + namespace='oauth2_provider')), # main website views re_path(r'^$', main_views.home), re_path(r'^tutorial/$', main_views.tutorial, name='tutorial'), re_path(r'^about-us/$', main_views.about_us, name='about-us'), re_path(r'^getting_started/$', main_views.getting_started, - name='getting_started'), + name='getting_started'), re_path(r'^faq/$', main_views.faq, name='faq'), re_path(r'^syntax/$', main_views.syntax, name='syntax'), re_path(r'^privacy/$', main_views.privacy, name='privacy'), @@ -74,246 +74,253 @@ re_path(r'^xls2xform/$', main_views.xls2xform), re_path(r'^support/$', main_views.support, name='support'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/stats$', - viewer_views.charts, name='form-stats'), + viewer_views.charts, name='form-stats'), re_path(r'^login_redirect/$', main_views.login_redirect), re_path(r'^attachment/$', viewer_views.attachment_url, - name='attachment_url'), + name='attachment_url'), re_path(r'^attachment/(?P[^/]+)$', viewer_views.attachment_url, - name='attachment_url'), + name='attachment_url'), re_path(r'^jsi18n/$', - django.views.i18n.JavaScriptCatalog.as_view( - packages=['onadata.apps.main', 'onadata.apps.viewer']), - name='javascript-catalog'), + django.views.i18n.JavaScriptCatalog.as_view( + packages=['onadata.apps.main', 'onadata.apps.viewer']), + name='javascript-catalog'), re_path(r'^typeahead_usernames', main_views.username_list, - name='username_list'), + name='username_list'), re_path(r'^(?P[^/]+)/$', main_views.profile, - name='user_profile'), + name='user_profile'), re_path(r'^(?P[^/]+)/profile$', main_views.public_profile, - name='public_profile'), + name='public_profile'), re_path(r'^(?P[^/]+)/settings', main_views.profile_settings, - name='profile-settings'), + name='profile-settings'), re_path(r'^(?P[^/]+)/cloneform$', main_views.clone_xlsform, - name='clone-xlsform'), + name='clone-xlsform'), re_path(r'^(?P[^/]+)/activity$', main_views.activity, - name='activity'), + name='activity'), re_path(r'^(?P[^/]+)/activity/api$', main_views.activity_api, - name='activity-api'), + name='activity-api'), re_path(r'^activity/fields$', main_views.activity_fields, - name='activity-fields'), + name='activity-fields'), re_path(r'^(?P[^/]+)/api-token$', main_views.api_token, - name='api-token'), + name='api-token'), # form specific re_path(r'^(?P[^/]+)/forms/(?P[^/]+)$', - main_views.show, - name='form-show'), + main_views.show, + name='form-show'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/qrcode$', - main_views.qrcode, name='get_qrcode'), + main_views.qrcode, name='get_qrcode'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/api$', - main_views.api, name='mongo_view_api'), + main_views.api, name='mongo_view_api'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/public_api$', - main_views.public_api, name='public_api'), + main_views.public_api, name='public_api'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete_data$', - main_views.delete_data, name='delete_data'), + main_views.delete_data, name='delete_data'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/edit$', - main_views.edit, name='xform-edit'), + main_views.edit, name='xform-edit'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/perms$', - main_views.set_perm, name='set-xform-permissions'), + main_views.set_perm, name='set-xform-permissions'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/photos', - main_views.form_photos, name='form-photos'), + main_views.form_photos, name='form-photos'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/doc/(?P\d+)', # noqa - main_views.download_metadata, name='download-metadata'), + main_views.download_metadata, name='download-metadata'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delete-doc/(?P\d+)', main_views.delete_metadata, # noqa - name='delete-metadata'), + name='delete-metadata'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)', main_views.download_media_data, # noqa - name='download-media-data'), + name='download-media-data'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/addservice$', - restservice_views.add_service, name='add_restservice'), + restservice_views.add_service, name='add_restservice'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/delservice$', - restservice_views.delete_service, - name='delete_restservice'), + restservice_views.delete_service, + name='delete_restservice'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/update$', - main_views.update_xform, name='update-xform'), + main_views.update_xform, name='update-xform'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/preview$', - main_views.enketo_preview, name='enketo-preview'), + main_views.enketo_preview, name='enketo-preview'), # briefcase api urls re_path(r'^(?P\w+)/view/submissionList$', - BriefcaseViewset.as_view({'get': 'list', 'head': 'list'}), - name='view-submission-list'), + BriefcaseViewset.as_view({'get': 'list', 'head': 'list'}), + name='view-submission-list'), re_path(r'^(?P\w+)/view/downloadSubmission$', - BriefcaseViewset.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='view-download-submission'), + BriefcaseViewset.as_view({'get': 'retrieve', 'head': 'retrieve'}), + name='view-download-submission'), re_path(r'^(?P\w+)/formUpload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='form-upload'), + BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), + name='form-upload'), re_path(r'^(?P\w+)/upload$', - BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), - name='upload'), + BriefcaseViewset.as_view({'post': 'create', 'head': 'create'}), + name='upload'), # exporting stuff re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv$', - viewer_views.data_export, name='csv_export', - kwargs={'export_type': 'csv'}), + viewer_views.data_export, name='csv_export', + kwargs={'export_type': 'csv'}), re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.xls', - viewer_views.data_export, name='xls_export', - kwargs={'export_type': 'xls'}), + viewer_views.data_export, name='xls_export', + kwargs={'export_type': 'xls'}), re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.csv.zip', - viewer_views.data_export, name='csv_zip_export', - kwargs={'export_type': 'csv_zip'}), + viewer_views.data_export, name='csv_zip_export', + kwargs={'export_type': 'csv_zip'}), re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.sav.zip', - viewer_views.data_export, name='sav_zip_export', - kwargs={'export_type': 'sav_zip'}), + viewer_views.data_export, name='sav_zip_export', + kwargs={'export_type': 'sav_zip'}), re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.kml$', - viewer_views.kml_export, name='kml-export'), + viewer_views.kml_export, name='kml-export'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/data\.zip', - viewer_views.zip_export), + viewer_views.zip_export), re_path(r'^(?P\w+)/forms/(?P[^/]+)/gdocs$', - viewer_views.google_xls_export), + viewer_views.google_xls_export), re_path(r'^(?P\w+)/forms/(?P[^/]+)/map_embed', - viewer_views.map_embed_view), + viewer_views.map_embed_view), re_path(r'^(?P\w+)/forms/(?P[^/]+)/map', - viewer_views.map_view, name='map-view'), + viewer_views.map_view, name='map-view'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/instance', - viewer_views.instance, name='submission-instance'), + viewer_views.instance, name='submission-instance'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/enter-data', - logger_views.enter_data, name='enter_data'), + logger_views.enter_data, name='enter_data'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/add-submission-with', # noqa - viewer_views.add_submission_with, - name='add_submission_with'), + viewer_views.add_submission_with, + name='add_submission_with'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/thank_you_submission', # noqa - viewer_views.thank_you_submission, - name='thank_you_submission'), + viewer_views.thank_you_submission, + name='thank_you_submission'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/edit-data/(?P\d+)$', # noqa - logger_views.edit_data, name='edit_data'), + logger_views.edit_data, name='edit_data'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/view-data', - viewer_views.data_view, name='data-view'), + viewer_views.data_view, name='data-view'), re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)/new$', # noqa - viewer_views.create_export, name='new-export'), + viewer_views.create_export, name='new-export'), re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/delete$', viewer_views.delete_export, name='delete-export'), + '/delete$', viewer_views.delete_export, name='delete-export'), re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/progress$', viewer_views.export_progress, name='export-progress'), + '/progress$', viewer_views.export_progress, + name='export-progress'), re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/$', viewer_views.export_list, name='export-list'), + '/$', viewer_views.export_list, name='export-list'), re_path(r'^(?P\w+)/exports/(?P[^/]+)/(?P\w+)' # noqa - '/(?P[^/]+)$', - viewer_views.export_download, name='export-download'), + '/(?P[^/]+)$', + viewer_views.export_download, name='export-download'), # open id connect urls re_path(r'^oidc/(?P\w+)/login$', - OpenIDConnectViewSet.as_view({ - 'get': 'initiate_oidc_flow', 'head': 'callback', 'post': 'callback' - }), name='open-id-connect-login'), + OpenIDConnectViewSet.as_view({ + 'get': 'initiate_oidc_flow', + 'head': 'callback', + 'post': 'callback' + }), name='open-id-connect-login'), re_path(r'^oidc/(?P\w+)/expire$', - OpenIDConnectViewSet.as_view({ - 'get': 'expire' - }), name='open-id-connect-logout'), + OpenIDConnectViewSet.as_view({ + 'get': 'expire' + }), name='open-id-connect-logout'), re_path(r'^oidc/(?P\w+)/callback$', - OpenIDConnectViewSet.as_view({ - 'get': 'callback', 'head': 'callback', 'post': 'callback' - }), name='open-id-connect-callback'), + OpenIDConnectViewSet.as_view({ + 'get': 'callback', 'head': 'callback', 'post': 'callback' + }), name='open-id-connect-callback'), # odk data urls re_path(r'^submission$', - XFormSubmissionViewSet.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), + XFormSubmissionViewSet.as_view( + {'post': 'create', 'head': 'create'}), + name='submissions'), re_path(r'^formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^enketo/(?P\w+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^(?P\w+)/(?P\d+)/formList$', - XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + XFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^preview/(?P\w+)/(?P\d+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^preview/(?P\w+)/formList$', - PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), - name='form-list'), + PreviewXFormListViewSet.as_view({'get': 'list', 'head': 'list'}), + name='form-list'), re_path(r'^(?P\w+)/xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), + XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), + name='manifest-url'), re_path(r'^xformsManifest/(?P[\d+^/]+)$', - XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), - name='manifest-url'), + XFormListViewSet.as_view({'get': 'manifest', 'head': 'manifest'}), + name='manifest-url'), re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', # noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), + XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), + name='xform-media'), re_path(r'^(?P\w+)/xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.(?P([a-z]|[0-9])*)$',# noqa - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), + XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), + name='xform-media'), re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), + XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), + name='xform-media'), re_path(r'^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\.' - '(?P([a-z]|[0-9])*)$', - XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), - name='xform-media'), + '(?P([a-z]|[0-9])*)$', + XFormListViewSet.as_view({'get': 'media', 'head': 'media'}), + name='xform-media'), re_path(r'^(?P\w+)/submission$', - XFormSubmissionViewSet.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), + XFormSubmissionViewSet.as_view( + {'post': 'create', 'head': 'create'}), + name='submissions'), re_path(r'^enketo/(?P\w+)/submission$', - XFormSubmissionViewSet.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), + XFormSubmissionViewSet.as_view( + {'post': 'create', 'head': 'create'}), + name='submissions'), re_path(r'^(?P\w+)/(?P\d+)/submission$', - XFormSubmissionViewSet.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), + XFormSubmissionViewSet.as_view( + {'post': 'create', 'head': 'create'}), + name='submissions'), re_path(r'^(?P\w+)/bulk-submission$', - logger_views.bulksubmission), + logger_views.bulksubmission), re_path(r'^(?P\w+)/bulk-submission-form$', - logger_views.bulksubmission_form), + logger_views.bulksubmission_form), re_path(r'^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$', - XFormListViewSet.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='download_xform'), + XFormListViewSet.as_view({'get': 'retrieve', 'head': 'retrieve'}), + name='download_xform'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xml$', - logger_views.download_xform, name='download_xform'), + logger_views.download_xform, name='download_xform'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.xls$', - logger_views.download_xlsform, - name='download_xlsform'), + logger_views.download_xlsform, + name='download_xlsform'), re_path(r'^(?P\w+)/forms/(?P[^/]+)/form\.json', - logger_views.download_jsonform, - name='download_jsonform'), + logger_views.download_jsonform, + name='download_jsonform'), re_path(r'^(?P\w+)/delete/(?P[^/]+)/$', - logger_views.delete_xform, name='delete-xform'), + logger_views.delete_xform, name='delete-xform'), re_path(r'^(?P\w+)/(?P[^/]+)/toggle_downloadable/$', - logger_views.toggle_downloadable, name='toggle-downloadable'), + logger_views.toggle_downloadable, name='toggle-downloadable'), # SMS support re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', # noqa - sms_support.providers.import_submission_for_form, - name='sms_submission_form_api'), + sms_support.providers.import_submission_for_form, + name='sms_submission_form_api'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_submission$', - sms_support_views.import_submission_for_form, - name='sms_submission_form'), + sms_support_views.import_submission_for_form, + name='sms_submission_form'), re_path(r'^(?P[^/]+)/sms_submission/(?P[a-z]+)/?$', - sms_support.providers.import_submission, - name='sms_submission_api'), + sms_support.providers.import_submission, + name='sms_submission_api'), re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/sms_multiple_submissions$', # noqa - sms_support_views.import_multiple_submissions_for_form, - name='sms_submissions_form'), + sms_support_views.import_multiple_submissions_for_form, + name='sms_submissions_form'), re_path(r'^(?P[^/]+)/sms_multiple_submissions$', - sms_support_views.import_multiple_submissions, - name='sms_submissions'), + sms_support_views.import_multiple_submissions, + name='sms_submissions'), re_path(r'^(?P[^/]+)/sms_submission$', - sms_support_views.import_submission, name='sms_submission'), + sms_support_views.import_submission, name='sms_submission'), # Stats tables re_path(r'^(?P\w+)/forms/(?P[^/]+)/tables', - viewer_views.stats_tables, name='stats-tables'), + viewer_views.stats_tables, name='stats-tables'), # static media re_path(r'^media/(?P.*)$', django.views.static.serve, - {'document_root': settings.MEDIA_ROOT}), + {'document_root': settings.MEDIA_ROOT}), re_path(r'^favicon\.ico', - RedirectView.as_view(url='/static/images/favicon.ico', - permanent=True)), + RedirectView.as_view(url='/static/images/favicon.ico', + permanent=True)), re_path(r'^static/(?P.*)$', staticfiles_views.serve) ] diff --git a/onadata/apps/viewer/parsed_instance_tools.py b/onadata/apps/viewer/parsed_instance_tools.py index 274c358c98..bc2d3a534c 100644 --- a/onadata/apps/viewer/parsed_instance_tools.py +++ b/onadata/apps/viewer/parsed_instance_tools.py @@ -104,8 +104,8 @@ def get_where_clause(query, form_integer_fields=None, if isinstance(query, dict) and '$or' in list(query): or_dict = query.pop('$or') - for l in or_dict: - for k, v in l.items(): + for or_query in or_dict: + for k, v in or_query.items(): if v is None: or_where.extend([u"json->>'{}' IS NULL".format(k)]) else: diff --git a/onadata/libs/data/query.py b/onadata/libs/data/query.py index 0668acc35f..f1f4bc6016 100644 --- a/onadata/libs/data/query.py +++ b/onadata/libs/data/query.py @@ -193,8 +193,8 @@ def _select_key(field, name, xform): return result -def flatten(l): - return [item for sublist in l for item in sublist] +def flatten(lst): + return [item for sublist in lst for item in sublist] def get_date_fields(xform): diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index 0b5d6e7bed..34c048311a 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -52,8 +52,8 @@ NO = 0 -def remove_dups_from_list_maintain_order(l): - return list(OrderedDict.fromkeys(l)) +def remove_dups_from_list_maintain_order(lst): + return list(OrderedDict.fromkeys(lst)) def get_prefix_from_xpath(xpath): From 1b2af91388f9cb75a7f98e450a2366f48aeff7c7 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 8 Oct 2020 14:45:58 +0300 Subject: [PATCH 4/6] Use celery application to register tasks --- onadata/apps/api/tasks.py | 12 ++++++------ onadata/apps/logger/import_tools.py | 4 ++-- onadata/apps/logger/models/instance.py | 8 ++++---- onadata/apps/messaging/tasks.py | 5 ++--- onadata/apps/restservice/tasks.py | 5 ++--- onadata/apps/viewer/tasks.py | 26 +++++++++++++------------- onadata/libs/utils/csv_import.py | 5 +++-- onadata/libs/utils/osm.py | 4 ++-- onadata/libs/utils/project_utils.py | 4 ++-- 9 files changed, 36 insertions(+), 37 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index 75bc3a66a9..bc013876d9 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -2,7 +2,6 @@ import sys from builtins import str -from celery import task from celery.result import AsyncResult from django.conf import settings from django.core.files.uploadedfile import TemporaryUploadedFile @@ -14,6 +13,7 @@ from onadata.apps.api import tools from onadata.apps.logger.models.xform import XForm +from onadata.celery import app def recreate_tmp_file(name, path, mime_type): @@ -23,7 +23,7 @@ def recreate_tmp_file(name, path, mime_type): return tmp_file -@task(bind=True) +@app.task(bind=True) def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): try: files = MultiValueDict() @@ -56,7 +56,7 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): return {u'error': error_message} -@task() +@app.task() def delete_xform_async(xform_id, user_id): """Soft delete an XForm asynchrounous task""" xform = XForm.objects.get(pk=xform_id) @@ -64,7 +64,7 @@ def delete_xform_async(xform_id, user_id): xform.soft_delete(user) -@task() +@app.task() def delete_user_async(): """Delete inactive user accounts""" users = User.objects.filter(active=False, @@ -105,7 +105,7 @@ def send_generic_email(email, message_txt, subject): email_message.send() -@task() +@app.task() def send_verification_email(email, message_txt, subject): """ Sends a verification email @@ -113,6 +113,6 @@ def send_verification_email(email, message_txt, subject): send_generic_email(email, message_txt, subject) -@task() +@app.task() def send_account_lockout_email(email, message_txt, subject): send_generic_email(email, message_txt, subject) diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index e8d2661448..4b0b017f95 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -5,10 +5,10 @@ import zipfile from builtins import open -from celery import task from django.core.files.uploadedfile import InMemoryUploadedFile from onadata.apps.logger.xform_fs import XFormInstanceFS +from onadata.celery import app from onadata.libs.utils.logger_tools import create_instance # odk @@ -74,7 +74,7 @@ def import_instance(username, xform_path, photos, osm_files, status, return 0 -@task(ignore_result=True) +@app.task(ignore_result=True) def import_instance_async(username, xform_path, photos, osm_files, status): import_instance(username, xform_path, photos, osm_files, status, False) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index f5ec17d352..a2d713e60e 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -5,7 +5,6 @@ import math from datetime import datetime -from celery import task from django.conf import settings from django.contrib.auth.models import User from django.contrib.gis.db import models @@ -27,6 +26,7 @@ from onadata.apps.logger.xform_instance_parser import (XFormInstanceParser, clean_and_parse_xml, get_uuid_from_xml) +from onadata.celery import app from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import (DATAVIEW_COUNT, IS_ORG, PROJ_NUM_DATASET_CACHE, @@ -143,7 +143,7 @@ def submission_time(): return timezone.now() -@task +@app.task @transaction.atomic() def update_xform_submission_count(instance_id, created): if created: @@ -215,7 +215,7 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): xform.save() -@task +@app.task def save_full_json(instance_id, created): """set json data, ensure the primary key is part of the json data""" if created: @@ -228,7 +228,7 @@ def save_full_json(instance_id, created): instance.save(update_fields=['json']) -@task +@app.task def update_project_date_modified(instance_id, created): # update the date modified field of the project which will change # the etag value of the projects endpoint diff --git a/onadata/apps/messaging/tasks.py b/onadata/apps/messaging/tasks.py index 5c9156a333..98a3b9c3e7 100644 --- a/onadata/apps/messaging/tasks.py +++ b/onadata/apps/messaging/tasks.py @@ -4,12 +4,11 @@ """ from __future__ import unicode_literals -from celery import task - from onadata.apps.messaging.backends.base import call_backend +from onadata.celery import app -@task(ignore_result=True) +@app.task(ignore_result=True) def call_backend_async(backend, instance_id, backend_options=None): """ Task to send messages to notification backeds such as MQTT diff --git a/onadata/apps/restservice/tasks.py b/onadata/apps/restservice/tasks.py index fa66a41530..b1895cc2bf 100644 --- a/onadata/apps/restservice/tasks.py +++ b/onadata/apps/restservice/tasks.py @@ -1,9 +1,8 @@ -from celery import task - from onadata.apps.restservice.utils import call_service +from onadata.celery import app -@task() +@app.task() def call_service_async(instance_pk): # load the parsed instance from onadata.apps.logger.models.instance import Instance diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index ce69480162..54a846e38c 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -11,7 +11,6 @@ from django.shortcuts import get_object_or_404 from django.utils import timezone -from celery import task from kombu.exceptions import OperationalError from requests import ConnectionError @@ -23,6 +22,7 @@ generate_external_export, generate_kml_export, generate_osm_export) +from onadata.celery import app EXPORT_QUERY_KEY = 'query' @@ -121,7 +121,7 @@ def _create_export(xform, export_type, options): return None -@task(track_started=True) +@app.task(track_started=True) def create_xls_export(username, id_string, export_id, **options): """ XLS export task. @@ -159,7 +159,7 @@ def create_xls_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_csv_export(username, id_string, export_id, **options): """ CSV export task. @@ -194,7 +194,7 @@ def create_csv_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_kml_export(username, id_string, export_id, **options): """ KML export task. @@ -227,7 +227,7 @@ def create_kml_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_osm_export(username, id_string, export_id, **options): """ OSM export task. @@ -261,7 +261,7 @@ def create_osm_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_zip_export(username, id_string, export_id, **options): """ Attachments zip export task. @@ -293,7 +293,7 @@ def create_zip_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_csv_zip_export(username, id_string, export_id, **options): """ CSV zip export task. @@ -320,7 +320,7 @@ def create_csv_zip_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_sav_zip_export(username, id_string, export_id, **options): """ SPSS sav export task. @@ -346,7 +346,7 @@ def create_sav_zip_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_external_export(username, id_string, export_id, **options): """ XLSReport export task. @@ -376,7 +376,7 @@ def create_external_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def create_google_sheet_export(username, id_string, export_id, **options): """ Google Sheets export task. @@ -403,7 +403,7 @@ def create_google_sheet_export(username, id_string, export_id, **options): return gen_export.id -@task(track_started=True) +@app.task(track_started=True) def delete_export(export_id): """ Delete export task with id export_id. @@ -418,7 +418,7 @@ def delete_export(export_id): return False -@task(ignore_result=True) +@app.task(ignore_result=True) def mark_expired_pending_exports_as_failed(): # pylint: disable=invalid-name """ Exports that have not completed within a set time should be marked as @@ -431,7 +431,7 @@ def mark_expired_pending_exports_as_failed(): # pylint: disable=invalid-name exports.update(internal_status=Export.FAILED) -@task(ignore_result=True) +@app.task(ignore_result=True) def delete_expired_failed_exports(): """ Delete old failed exports diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 7ce3806cc9..68e917111e 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -16,7 +16,7 @@ import unicodecsv as ucsv import xlrd -from celery import current_task, task +from celery import current_task from celery.backends.amqp import BacklogLimitExceeded from celery.result import AsyncResult from dateutil.parser import parse @@ -31,6 +31,7 @@ from onadata.apps.logger.models import Instance, XForm from onadata.apps.messaging.constants import XFORM, SUBMISSION_DELETED from onadata.apps.messaging.serializers import send_message +from onadata.celery import app from onadata.libs.utils import analytics from onadata.libs.utils.async_status import (FAILED, async_status, celery_state_to_status) @@ -142,7 +143,7 @@ def dict_pathkeys_to_nested_dicts(dictionary): return data -@task() +@app.task() def submit_csv_async(username, xform_id, file_path, overwrite=False): """Imports CSV data to an existing xform asynchrounously.""" xform = XForm.objects.get(pk=xform_id) diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index 48e7805cac..23393476c1 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -6,7 +6,6 @@ import logging -from celery import task from django.contrib.gis.geos import (GeometryCollection, LineString, Point, Polygon) from django.contrib.gis.geos.error import GEOSException @@ -18,6 +17,7 @@ from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.restservice.signals import trigger_webhook +from onadata.celery import app def _get_xml_obj(xml): @@ -140,7 +140,7 @@ def parse_osm(osm_xml, include_osm_id=False): return nodes -@task() +@app.task() def save_osm_data_async(instance_id): """ Async task for saving OSM data for the specified submission. diff --git a/onadata/libs/utils/project_utils.py b/onadata/libs/utils/project_utils.py index 9035565175..3248c05c58 100644 --- a/onadata/libs/utils/project_utils.py +++ b/onadata/libs/utils/project_utils.py @@ -4,11 +4,11 @@ """ import sys -from celery import task from django.conf import settings from django.db import IntegrityError from onadata.apps.logger.models import Project, XForm +from onadata.celery import app from onadata.libs.permissions import (ROLES, OwnerRole, get_object_users_with_permissions) from onadata.libs.utils.common_tags import OWNER_TEAM_NAME @@ -59,7 +59,7 @@ def set_project_perms_to_xform(xform, project): # pylint: disable=invalid-name -@task(bind=True, max_retries=3) +@app.task(bind=True, max_retries=3) def set_project_perms_to_xform_async(self, xform_id, project_id): """ Apply project permissions for ``project_id`` to a form ``xform_id`` task. From e7bf3d3adade7540a20facc400cfb363d9c47334 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 8 Oct 2020 14:46:29 +0300 Subject: [PATCH 5/6] Remove usage of celery amqp backend Celerys amqp backend was removed in v5.0.0 --- onadata/libs/tests/utils/test_api_export_tools.py | 2 +- onadata/libs/tests/utils/test_csv_import.py | 2 +- onadata/libs/utils/api_export_tools.py | 2 +- onadata/libs/utils/csv_import.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index 34c406abf8..9f1bb9736c 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -5,7 +5,7 @@ from collections import OrderedDict, defaultdict import mock -from celery.backends.amqp import BacklogLimitExceeded +from celery.backends.rpc import BacklogLimitExceeded from django.http import Http404 from django.test.utils import override_settings from kombu.exceptions import OperationalError diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index 9adfb9fca2..ccd618fbd7 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -8,7 +8,7 @@ import mock import unicodecsv as ucsv -from celery.backends.amqp import BacklogLimitExceeded +from celery.backends.rpc import BacklogLimitExceeded from django.conf import settings from mock import patch diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 33afd2cd0d..ef855641da 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -8,7 +8,7 @@ from datetime import datetime import httplib2 -from celery.backends.amqp import BacklogLimitExceeded +from celery.backends.rpc import BacklogLimitExceeded from celery.result import AsyncResult from django.conf import settings from django.http import Http404, HttpResponseRedirect diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 68e917111e..97980e71ae 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -17,7 +17,7 @@ import unicodecsv as ucsv import xlrd from celery import current_task -from celery.backends.amqp import BacklogLimitExceeded +from celery.backends.rpc import BacklogLimitExceeded from celery.result import AsyncResult from dateutil.parser import parse from django.conf import settings From 2fdddafbf0d9da3b6c00841095ffac9d90becc97 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 8 Oct 2020 14:47:58 +0300 Subject: [PATCH 6/6] Remove submitted_by__id & submitted_by__username from Meta.fields --- onadata/libs/filters.py | 2 -- onadata/settings/common.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 4680fd9ada..c77d992d6e 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -175,8 +175,6 @@ class Meta: 'last_edited': date_field_lookups, 'media_all_received': ['exact'], 'status': ['exact'], - 'submitted_by__id': ['exact'], - 'submitted_by__username': ['exact'], 'survey_type__slug': ['exact'], 'user__id': ['exact'], 'user__username': ['exact'], diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 65b76af2c8..8df3b81a61 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -211,7 +211,7 @@ 'reversion', 'actstream', 'onadata.apps.messaging.apps.MessagingConfig', - 'django_celery_results', + 'django_filters', ) OAUTH2_PROVIDER = {