Merge pull request #3573 from hotosm/develop

v4.1.8 release

backend/__init__.py

@@ -34,7 +34,9 @@ def create_app(env=None):
     :return: Initialised Flask app
     """
 
-    app = Flask(__name__,)
+    app = Flask(
+        __name__,
+    )
 
     # Load configuration options from environment
     app.config.from_object("backend.config.EnvironmentConfig")
@@ -754,7 +756,9 @@ def add_api_endpoints(app):
         SystemAuthenticationEmailAPI, format_url("system/authentication/email/")
     )
     api.add_resource(
-        SystemImageUploadRestAPI, format_url("system/image-upload/"), methods=["POST"],
+        SystemImageUploadRestAPI,
+        format_url("system/image-upload/"),
+        methods=["POST"],
     )
     api.add_resource(
         SystemApplicationsRestAPI,

backend/api/issues/resources.py

@@ -93,8 +93,8 @@ def patch(self, category_id):
             return {"Error": "Unable to update mapping issue category"}, 400
 
         try:
-            updated_category = MappingIssueCategoryService.update_mapping_issue_category(
-                category_dto
+            updated_category = (
+                MappingIssueCategoryService.update_mapping_issue_category(category_dto)
             )
             return updated_category.to_primitive(), 200
         except NotFound:

backend/api/organisations/campaigns.py

@@ -53,16 +53,20 @@ def post(self, organisation_id, campaign_id):
                 organisation_id, token_auth.current_user()
             ):
                 if Campaign.campaign_organisation_exists(campaign_id, organisation_id):
-                    message = "Campaign {} is already assigned to organisation {}.".format(
-                        campaign_id, organisation_id
+                    message = (
+                        "Campaign {} is already assigned to organisation {}.".format(
+                            campaign_id, organisation_id
+                        )
                     )
                     return {"Error": message}, 400
 
                 CampaignService.create_campaign_organisation(
                     organisation_id, campaign_id
                 )
-                message = "campaign with id {} assigned for organisation with id {}".format(
-                    campaign_id, organisation_id
+                message = (
+                    "campaign with id {} assigned for organisation with id {}".format(
+                        campaign_id, organisation_id
+                    )
                 )
                 return {"Success": message}, 200
             else:

backend/api/projects/teams.py

@@ -8,7 +8,7 @@
 class ProjectsTeamsAPI(Resource):
     @token_auth.login_required
     def get(self, project_id):
-        """ Get teams assigned with a project
+        """Get teams assigned with a project
         ---
         tags:
           - teams
@@ -47,7 +47,7 @@ def get(self, project_id):
 
     @token_auth.login_required
     def post(self, team_id, project_id):
-        """ Assign a team to a project
+        """Assign a team to a project
         ---
         tags:
           - teams
@@ -118,7 +118,7 @@ def post(self, team_id, project_id):
 
     @token_auth.login_required
     def patch(self, team_id, project_id):
-        """ Update role of a team on a project
+        """Update role of a team on a project
         ---
         tags:
           - teams

backend/config.py

@@ -36,6 +36,12 @@ class EnvironmentConfig:
     # A freely definable secret key for connecting the front end with the back end
     SECRET_KEY = os.getenv("TM_SECRET", None)
 
+    # OSM API, Nomimatim URLs
+    OSM_SERVER_URL = os.getenv("OSM_SERVER_URL", "https://www.openstreetmap.org")
+    OSM_NOMINATIM_SERVER_URL = os.getenv(
+        "OSM_NOMINATIM_SERVER_URL", "https://nominatim.openstreetmap.org"
+    )
+
     # Database connection
     POSTGRES_USER = os.getenv("POSTGRES_USER", None)
     POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", None)
@@ -89,12 +95,12 @@ class EnvironmentConfig:
 
     # Connection to OSM authentification system
     OSM_OAUTH_SETTINGS = {
-        "base_url": "https://www.openstreetmap.org/api/0.6/",
+        "base_url": "{}/api/0.6/".format(OSM_SERVER_URL),
         "consumer_key": os.getenv("TM_CONSUMER_KEY", None),
         "consumer_secret": os.getenv("TM_CONSUMER_SECRET", None),
-        "request_token_url": "https://www.openstreetmap.org/oauth/request_token",
-        "access_token_url": "https://www.openstreetmap.org/oauth/access_token",
-        "authorize_url": "https://www.openstreetmap.org/oauth/authorize",
+        "request_token_url": "{}/oauth/request_token".format(OSM_SERVER_URL),
+        "access_token_url": "{}/oauth/access_token".format(OSM_SERVER_URL),
+        "authorize_url": "{}/oauth/authorize".format(OSM_SERVER_URL),
     }
 
     # Some more definitions (not overridable)

backend/models/postgis/organisation.py

@@ -123,15 +123,15 @@ def get(organisation_id: int):
 
     @staticmethod
     def get_organisation_by_name(organisation_name: str):
-        """ Get organisation by name
+        """Get organisation by name
         :param organisation_name: name of organisation
         :return: Organisation if found else None
         """
         return Organisation.query.filter_by(name=organisation_name).first()
 
     @staticmethod
     def get_organisation_name_by_id(organisation_id: int):
-        """ Get organisation name by id
+        """Get organisation name by id
         :param organisation_id:
         :return: Organisation name
         """

backend/models/postgis/project.py

@@ -7,7 +7,7 @@
 import datetime
 from flask import current_app
 from geoalchemy2 import Geometry
-import sqlalchemy
+from geoalchemy2.shape import to_shape
 from sqlalchemy.sql.expression import cast, or_
 from sqlalchemy import text, desc, func, Time, orm, literal
 from shapely.geometry import shape
@@ -57,8 +57,6 @@
     timestamp,
     ST_Centroid,
     NotFound,
-    ST_X,
-    ST_Y,
 )
 from backend.services.grid.grid_service import GridService
 from backend.models.postgis.interests import Interest, project_interests
@@ -260,25 +258,17 @@ def set_default_changeset_comment(self):
     def set_country_info(self):
         """ Sets the default country based on centroid"""
 
-        lat, lng = (
-            db.session.query(
-                cast(ST_Y(Project.centroid), sqlalchemy.String),
-                cast(ST_X(Project.centroid), sqlalchemy.String),
-            )
-            .filter(Project.id == self.id)
-            .one()
+        centroid = to_shape(self.centroid)
+        lat, lng = (centroid.y, centroid.x)
+        url = "{0}/reverse?format=jsonv2&lat={1}&lon={2}&accept-language=en".format(
+            current_app.config["OSM_NOMINATIM_SERVER_URL"], lat, lng
         )
-        url = "https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={0}&lon={1}".format(
-            lat, lng
-        )
-        country_info = requests.get(url)
-        country_info_json = country_info.content.decode("utf8").replace("'", '"')
-        # Load the JSON to a Python list & dump it back out as formatted JSON
-        data = json.loads(country_info_json)
-        if data["address"].get("country") is not None:
-            self.country = [data["address"]["country"]]
-        else:
-            self.country = [data["address"]["county"]]
+        try:
+            country_info = requests.get(url).json()  # returns a dict
+            if country_info["address"].get("country") is not None:
+                self.country = [country_info["address"]["country"]]
+        except (KeyError, AttributeError, requests.exceptions.ConnectionError):
+            pass
 
         self.save()
 
@@ -474,6 +464,9 @@ def update(self, project_dto: ProjectDTO):
         self.interests = []
         if project_dto.interests:
             self.interests = [Interest.query.get(i.id) for i in project_dto.interests]
+        # try to update country info if that information is not present
+        if len(self.country) == 0:
+            self.set_country_info()
 
         db.session.commit()
 

backend/models/postgis/task.py

@@ -139,8 +139,8 @@ def record_validation(project_id, task_id, validator_id, history):
 
 
 class TaskMappingIssue(db.Model):
-    """ Describes an issue (along with an occurrence count) with a
-    task mapping that contributed to invalidation of the task """
+    """Describes an issue (along with an occurrence count) with a
+    task mapping that contributed to invalidation of the task"""
 
     __tablename__ = "task_mapping_issues"
     id = db.Column(db.Integer, primary_key=True)
@@ -294,9 +294,9 @@ def update_task_locked_with_duration(
     def remove_duplicate_task_history_rows(
         task_id: int, project_id: int, lock_action: TaskStatus, user_id: int
     ):
-        """ Method used in rare cases where we have duplicate task history records for a given action by a user
-            This method will remove the oldest duplicate record, on the basis that the newest record was the
-            last action the user was attempting to perform
+        """Method used in rare cases where we have duplicate task history records for a given action by a user
+        This method will remove the oldest duplicate record, on the basis that the newest record was the
+        last action the user was attempting to perform
         """
         dupe = (
             TaskHistory.query.filter(

backend/models/postgis/user.py

@@ -2,6 +2,7 @@
 from backend import db
 from sqlalchemy import desc, func
 from geoalchemy2 import functions
+from flask import current_app
 from backend.models.dtos.user_dto import (
     UserDTO,
     UserMappedProjectsDTO,
@@ -73,7 +74,7 @@ def missing_maps_profile_url(self):
 
     @property
     def osm_profile_url(self):
-        return f"https://www.openstreetmap.org/user/{self.username}"
+        return f"{current_app.config['OSM_SERVER_URL']}/user/{self.username}"
 
     def create(self):
         """ Creates and saves the current model to the DB """
@@ -180,7 +181,7 @@ def get_all_users_not_paginated():
 
     @staticmethod
     def filter_users(user_filter: str, project_id: int, page: int) -> UserFilterDTO:
-        """ Finds users that matches first characters, for auto-complete.
+        """Finds users that matches first characters, for auto-complete.
 
         Users who have participated (mapped or validated) in the project, if given, will be
         returned ahead of those who have not.

backend/services/mapping_service.py

@@ -44,7 +44,9 @@ def get_task(task_id: int, project_id: int) -> Task:
 
     @staticmethod
     def get_task_as_dto(
-        task_id: int, project_id: int, preferred_local: str = "en",
+        task_id: int,
+        project_id: int,
+        preferred_local: str = "en",
     ) -> TaskDTO:
         """ Get task as DTO for transmission over API """
         task = MappingService.get_task(task_id, project_id)
@@ -262,8 +264,8 @@ def generate_gpx(project_id: int, task_ids_str: str, timestamp=None):
 
     @staticmethod
     def generate_osm_xml(project_id: int, task_ids_str: str) -> str:
-        """ Generate xml response suitable for loading into JOSM.  A sample output file is in
-            /backend/helpers/testfiles/osm-sample.xml """
+        """Generate xml response suitable for loading into JOSM.  A sample output file is in
+        /backend/helpers/testfiles/osm-sample.xml"""
         # Note XML created with upload No to ensure it will be rejected by OSM if uploaded by mistake
         root = ET.Element(
             "osm",

backend/services/messaging/message_service.py

@@ -106,8 +106,8 @@ def send_message_after_validation(
 
     @staticmethod
     def send_message_to_all_contributors(project_id: int, message_dto: MessageDTO):
-        """  Sends supplied message to all contributors on specified project.  Message all contributors can take
-             over a minute to run, so this method is expected to be called on its own thread """
+        """Sends supplied message to all contributors on specified project.  Message all contributors can take
+        over a minute to run, so this method is expected to be called on its own thread"""
 
         app = (
             create_app()

backend/services/messaging/smtp_service.py

@@ -46,7 +46,9 @@ def send_contact_admin_email(data):
         message = """New contact from {name} - {email}.
             <p>{content}</p>
             """.format(
-            name=data.get("name"), email=data.get("email"), content=data.get("content"),
+            name=data.get("name"),
+            email=data.get("email"),
+            content=data.get("content"),
         )
 
         subject = "New contact from {name}".format(name=data.get("name"))

backend/services/organisation_service.py

@@ -44,8 +44,10 @@ def get_organisation_by_id_as_dto(
         organisation_dto = org.as_dto(abbreviated)
 
         if user_id != 0:
-            organisation_dto.is_manager = OrganisationService.can_user_manage_organisation(
-                organisation_id, user_id
+            organisation_dto.is_manager = (
+                OrganisationService.can_user_manage_organisation(
+                    organisation_id, user_id
+                )
             )
         else:
             organisation_dto.is_manager = False

backend/services/project_service.py

@@ -205,8 +205,10 @@ def get_project_dto_for_mapper(
         is_manager_permission = False
 
         if current_user_id:
-            is_manager_permission = ProjectAdminService.is_user_action_permitted_on_project(
-                current_user_id, project_id
+            is_manager_permission = (
+                ProjectAdminService.is_user_action_permitted_on_project(
+                    current_user_id, project_id
+                )
             )
         # Draft Projects - admins, authors, org admins & team managers permitted
         if project.status == ProjectStatus.DRAFT.value:

backend/services/team_service.py

@@ -217,7 +217,9 @@ def get_all_teams(
             query = manager_teams.union(manager_orgs_teams)
 
         if team_name_filter:
-            query = query.filter(Team.name.ilike("%" + team_name_filter + "%"),)
+            query = query.filter(
+                Team.name.ilike("%" + team_name_filter + "%"),
+            )
 
         if team_role_filter:
             try:
@@ -476,7 +478,8 @@ def delete_invite(team_id: int, user_id: int):
     @staticmethod
     def is_user_team_member(team_id: int, user_id: int):
         query = TeamMembers.query.filter(
-            TeamMembers.team_id == team_id, TeamMembers.user_id == user_id,
+            TeamMembers.team_id == team_id,
+            TeamMembers.user_id == user_id,
         ).exists()
         return db.session.query(query).scalar()
 
@@ -538,8 +541,8 @@ def check_team_membership(project_id: int, allowed_roles: list, user_id: int):
     def send_message_to_all_team_members(
         team_id: int, team_name: str, message_dto: MessageDTO
     ):
-        """  Sends supplied message to all contributors in a team.  Message all team members can take
-             over a minute to run, so this method is expected to be called on its own thread """
+        """Sends supplied message to all contributors in a team.  Message all team members can take
+        over a minute to run, so this method is expected to be called on its own thread"""
         app = (
             create_app()
         )  # Because message-all run on background thread it needs it's own app context
@@ -548,10 +551,12 @@ def send_message_to_all_team_members(
             team_members = TeamService._get_active_team_members(team_id)
             sender = UserService.get_user_by_id(message_dto.from_user_id).username
 
-            message_dto.message = "A message from {}, manager of {} team:<br/><br/>{}".format(
-                MessageService.get_user_profile_link(sender),
-                MessageService.get_team_link(team_name, team_id, False),
-                markdown(message_dto.message, output_format="html"),
+            message_dto.message = (
+                "A message from {}, manager of {} team:<br/><br/>{}".format(
+                    MessageService.get_user_profile_link(sender),
+                    MessageService.get_team_link(team_name, team_id, False),
+                    markdown(message_dto.message, output_format="html"),
+                )
             )
 
             messages = []

backend/services/users/osm_service.py

@@ -21,7 +21,9 @@ def get_osm_details_for_user(user_id: int) -> UserOSMDTO:
         :param user_id: user_id in scope
         :raises OSMServiceError
         """
-        osm_user_details_url = f"http://www.openstreetmap.org/api/0.6/user/{user_id}"
+        osm_user_details_url = (
+            f"{current_app.config['OSM_SERVER_URL']}/api/0.6/user/{user_id}"
+        )
         response = requests.get(osm_user_details_url)
 
         if response.status_code != 200:

backend/services/users/user_service.py

@@ -427,8 +427,8 @@ def get_detailed_stats(username: str):
 
     @staticmethod
     def update_user_details(user_id: int, user_dto: UserDTO) -> dict:
-        """ Update user with info supplied by user, if they add or change their email address a verification mail
-            will be sent """
+        """Update user with info supplied by user, if they add or change their email address a verification mail
+        will be sent"""
         user = UserService.get_user_by_id(user_id)
 
         verification_email_sent = False