Switch to prospector

.flake8

@@ -1,5 +1,5 @@
 [flake8]
 max-line-length = 88
 select = C,E,F,W,B,B950
-extend-ignore = E203, E501
+extend-ignore = E203,E501
 per-file-ignores = __init__.py:F401

.github/workflows/ci.yml

@@ -75,6 +75,14 @@ jobs:
           pip install mock
           pip install requests-mock
 
+      - name: Python Static Analysis
+        env:
+          DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
+        run: |
+          pip install prospector
+          pip install -r requirements/azure.pip
+          prospector -X -s veryhigh onadata
+
       - name: Run tests
         run: |
           python manage.py test ${{ matrix.testfolder }} --noinput --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    -   id: black

.prospector.yaml

@@ -5,10 +5,14 @@ autodetect: true
 member-warnings: false
 max-line-length: 88
 
+pycodestyle:
+  disable:
+    - E203  # Whitespace before ':'
 pylint:
   options:
     extension-pkg-allow-list:
       - ujson
+      - lxml.etree
 
 mccabe:
   run: false

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-allow-list=ujson
+extension-pkg-allow-list=ujson,lxml.etree
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods
+disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods,django-not-configured
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

onadata/__init__.py

@@ -11,6 +11,6 @@
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
-from .celery import app as celery_app
+from .celeryapp import app as celery_app
 
 __all__ = ("celery_app",)

onadata/apps/api/management/commands/apply_can_add_project_perms.py

@@ -9,6 +9,7 @@
 """
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
+
 from guardian.shortcuts import assign_perm
 
 from onadata.apps.api.models import OrganizationProfile
@@ -24,9 +25,10 @@ def org_can_add_project_permission():
 
     for organization in organizations.iterator():
         permissions = organization.orgprofileuserobjectpermission_set.filter(
-            permission__codename='can_add_xform')
+            permission__codename="can_add_xform"
+        )
         for permission in permissions:
-            assign_perm('can_add_project', permission.user, organization)
+            assign_perm("can_add_project", permission.user, organization)
 
 
 def user_can_add_project_permission():
@@ -38,17 +40,19 @@ def user_can_add_project_permission():
 
     for user in users.iterator():
         permissions = user.userprofileuserobjectpermission_set.filter(
-            permission__codename='can_add_xform')
+            permission__codename="can_add_xform"
+        )
         for permission in permissions:
-            assign_perm('can_add_project', permission.user, user)
+            assign_perm("can_add_project", permission.user, user)
 
 
 class Command(BaseCommand):
     """
     Command apply_can_add_preject_perms - applys can_add_project permission to
     all users who have can_add_xform permission to a user/organization profile.
     """
-    help = _(u"Apply can_add_project permissions")
+
+    help = _("Apply can_add_project permissions")
 
     def handle(self, *args, **options):
         user_can_add_project_permission()

onadata/apps/api/management/commands/assign_team_member_permission.py

@@ -1,20 +1,26 @@
-from django.core.management.base import BaseCommand
+# -*- coding: utf-8 -*-
+"""
+Assign permission to the member team
+"""
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
-from onadata.apps.api.models.team import Team
+from guardian.shortcuts import assign_perm, get_perms_for_model
+
 from onadata.apps.api.models.organization_profile import OrganizationProfile
+from onadata.apps.api.models.team import Team
 from onadata.libs.utils.model_tools import queryset_iterator
 
-from guardian.shortcuts import assign_perm, get_perms_for_model
-
 
 class Command(BaseCommand):
-    args = '<organization>'
-    help = _(u"Assign permission to the member team")
+    """Assign permission to the member team"""
+
+    args = "<organization>"
+    help = _("Assign permission to the member team")
 
     def handle(self, *args, **options):
-        self.stdout.write("Assign permission to the member team", ending='\n')
+        self.stdout.write("Assign permission to the member team", ending="\n")
 
         count = 0
         fail = 0
@@ -24,38 +30,41 @@ def handle(self, *args, **options):
                 org_name = args[0]
                 org = OrganizationProfile.objects.get(user__username=org_name)
 
-                team = Team.objects.get(organization=org.user,
-                                        name=u'%s#%s' % (
-                                            org.user.username,
-                                            'members'))
+                team = Team.objects.get(
+                    organization=org.user, name=f"{org.user.username}#members" % ("")
+                )
                 self.assign_perm(team, org)
                 count += 1
                 total += 1
 
             except ObjectDoesNotExist as e:
                 fail += 1
-                self.stdout.write(str(e), ending='\n')
+                self.stdout.write(str(e), ending="\n")
         else:
             # Get all the teams
             for team in queryset_iterator(
-                    Team.objects.filter(name__contains='members')):
+                Team.objects.filter(name__contains="members")
+            ):
                 self.assign_perm(team, team.organization)
                 count += 1
                 total += 1
 
-        self.stdout.write("Assigned  {} of {} records. failed: {}".
-                          format(count, total, fail), ending='\n')
+        self.stdout.write(
+            f"Assigned  {count} of {total} records. failed: {fail}", ending="\n"
+        )
 
     def assign_perm(self, team, org):
+        """Assign a team org permissions"""
         for perm in get_perms_for_model(Team):
-            org = org.user \
-                if isinstance(org, OrganizationProfile) else org
+            org = org.user if isinstance(org, OrganizationProfile) else org
 
             assign_perm(perm.codename, org, team)
             if team.created_by:
                 assign_perm(perm.codename, team.created_by, team)
-            if hasattr(org.profile, 'creator') and \
-                    org.profile.creator != team.created_by:
-                assign_perm(perm.codename,  org.profile.creator, team)
+            if (
+                hasattr(org.profile, "creator")
+                and org.profile.creator != team.created_by
+            ):
+                assign_perm(perm.codename, org.profile.creator, team)
             if org.profile.created_by != team.created_by:
                 assign_perm(perm.codename, org.profile.created_by, team)

onadata/apps/api/management/commands/cleanup_permissions.py

@@ -1,3 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Cleanup permissions
+"""
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 from guardian.models import UserObjectPermission
@@ -7,32 +11,28 @@
 
 
 class Command(BaseCommand):
-    help = _(u"Cleanup permissions")
+    """Cleanup permissions"""
+
+    help = _("Cleanup permissions")
 
     def handle(self, *args, **options):
         deleted = 0
         self.stdout.write("Starting UserObject")
-        for perm in queryset_iterator(
-                UserObjectPermission.objects.select_related()):
+        for perm in queryset_iterator(UserObjectPermission.objects.select_related()):
             try:
                 perm.content_object
             except AttributeError:
                 perm.delete()
                 deleted += 1
-                self.stdout.write(
-                    "deleted {} stale permission".format(deleted)
-                )
+                self.stdout.write(f"deleted {deleted} stale permission")
         self.stdout.write("Starting GroupObject")
-        for perm in queryset_iterator(
-                GroupObjectPermission.objects.select_related()):
+        for perm in queryset_iterator(GroupObjectPermission.objects.select_related()):
             try:
                 perm.content_object
             except AttributeError:
                 perm.delete()
                 deleted += 1
-                self.stdout.write(
-                    "deleted {} stale permission".format(deleted)
-                )
+                self.stdout.write(f"deleted {deleted} stale permission")
         self.stdout.write(
-            "Total removed orphan object permissions instances: %d" % deleted
+            f"Total removed orphan object permissions instances: {deleted}"
         )

onadata/apps/api/management/commands/create_default_project.py

@@ -1,18 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+Check for forms not in a project and move them to the default project
+"""
 from django.conf import settings
+from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
-from django.contrib.auth.models import User
 from django.utils.translation import gettext as _
 
 from onadata.apps.logger.models.project import Project
 from onadata.libs.utils.model_tools import queryset_iterator
 
+User = get_user_model()
 
 XFORM_DEFAULT_PROJECT_ID = 1
 
 
 class Command(BaseCommand):
-    help = _(u"Check for forms not in a project"
-             u" and move them to the default project")
+    """
+    Check for forms not in a project and move them to the default project
+    """
+
+    help = _("Check for forms not in a project and move them to the default project")
 
     def handle(self, *args, **options):
         self.stdout.write("Task started ...")
@@ -27,16 +35,19 @@ def handle(self, *args, **options):
         self.stdout.write("Task completed ...")
 
     def set_project_to_user_forms(self, user):
-        default_project_name = user.username + '\'s Project'
+        """Set default project for all user forms."""
+        default_project_name = user.username + "'s Project"
         try:
             project = Project.objects.get(name=default_project_name)
         except Project.DoesNotExist:
-            metadata = {'description': 'Default Project'}
-            project = Project.objects.create(name=default_project_name,
-                                             organization=user,
-                                             created_by=user,
-                                             metadata=metadata)
-            self.stdout.write("Created project %s" % project.name)
+            metadata = {"description": "Default Project"}
+            project = Project.objects.create(
+                name=default_project_name,
+                organization=user,
+                created_by=user,
+                metadata=metadata,
+            )
+            self.stdout.write(f"Created project {project.name}")
         finally:
             xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID)
 

onadata/apps/api/management/commands/create_user_profiles.py

@@ -1,15 +1,18 @@
 # -*- coding: utf-8 -*-
 """Management Command to add missing user profiles to users."""
+from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
-from django.contrib.auth.models import User
 from django.utils.translation import gettext as _
 
 from onadata.apps.main.models.user_profile import UserProfile
 from onadata.libs.utils.model_tools import queryset_iterator
 
+User = get_user_model()
+
 
 class Command(BaseCommand):
     """Create missing user profiles management command."""
+
     help = _("Build out missing user profiles")
 
     def handle(self, *args, **options):