Merge pull request #3694 from hotosm/develop

v4.1.10 Release

.circleci/config.yml

@@ -162,6 +162,8 @@ jobs:
         type: env_var_name
       upload_api_key:
         type: env_var_name
+      sentry_backend_dsn: 
+        type: env_var_name
     working_directory: /home/circleci/tasking-manager
     docker:
     - image: circleci/python:3.7-buster-node
@@ -171,7 +173,7 @@ jobs:
     - run:
         name: Set Environment Variables
         command: |
-          echo "export JSON_CONFIG='{\"GitSha\":\"$CIRCLE_SHA1\", \"NetworkEnvironment\":\"<< parameters.environment_name >>\", \"AutoscalingPolicy\":\"<< parameters.autoscaling_policy >>\", \"DBSnapshot\":\"\", \"DatabaseDump\":\"\", \"NewRelicLicense\":\"${<< parameters.new_relic_license >>}\", \"PostgresDB\":\"${<< parameters.postgres_db >>}\", \"PostgresEndpoint\":\"\", \"PostgresPassword\":\"${<< parameters.postgres_password >>}\", \"PostgresUser\":\"${<< parameters.postgres_user >>}\", \"DatabaseSize\":\"${<< parameters.db_size >>}\",\"ELBSubnets\":\"${<< parameters.elb_subnets >>}\", \"SSLCertificateIdentifier\":\"${<< parameters.ssl_cert >>}\", \"TaskingManagerLogDirectory\":\"${<< parameters.log_dir >>}\", \"TaskingManagerConsumerKey\":\"${<< parameters.consumer_key >>}\",\"TaskingManagerConsumerSecret\":\"${<< parameters.consumer_secret >>}\",\"TaskingManagerSecret\":\"${<< parameters.tm_secret >>}\",\"TaskingManagerLogLevel\":\"<< parameters.log_level >>\",\"TaskingManagerImageUploadAPIURL\":\"${<< parameters.upload_api_url >>}\", \"TaskingManagerImageUploadAPIKey\":\"${<< parameters.upload_api_key >>}\",\"TaskingManagerURL\":\"${<< parameters.app_url >>}\", \"TaskingManagerOrgName\":\"${<< parameters.org_name >>}\", \"TaskingManagerOrgCode\":\"${<< parameters.org_code >>}\", \"TaskingManagerEmailContactAddress\":\"${<< parameters.email_contact_address >>}\"}'" >> $BASH_ENV
+          echo "export JSON_CONFIG='{\"GitSha\":\"$CIRCLE_SHA1\", \"NetworkEnvironment\":\"<< parameters.environment_name >>\", \"AutoscalingPolicy\":\"<< parameters.autoscaling_policy >>\", \"DBSnapshot\":\"\", \"DatabaseDump\":\"\", \"NewRelicLicense\":\"${<< parameters.new_relic_license >>}\", \"PostgresDB\":\"${<< parameters.postgres_db >>}\", \"PostgresEndpoint\":\"\", \"PostgresPassword\":\"${<< parameters.postgres_password >>}\", \"PostgresUser\":\"${<< parameters.postgres_user >>}\", \"DatabaseSize\":\"${<< parameters.db_size >>}\",\"ELBSubnets\":\"${<< parameters.elb_subnets >>}\", \"SSLCertificateIdentifier\":\"${<< parameters.ssl_cert >>}\", \"TaskingManagerLogDirectory\":\"${<< parameters.log_dir >>}\", \"TaskingManagerConsumerKey\":\"${<< parameters.consumer_key >>}\",\"TaskingManagerConsumerSecret\":\"${<< parameters.consumer_secret >>}\",\"TaskingManagerSecret\":\"${<< parameters.tm_secret >>}\",\"TaskingManagerLogLevel\":\"<< parameters.log_level >>\",\"TaskingManagerImageUploadAPIURL\":\"${<< parameters.upload_api_url >>}\", \"TaskingManagerImageUploadAPIKey\":\"${<< parameters.upload_api_key >>}\",\"TaskingManagerURL\":\"${<< parameters.app_url >>}\", \"TaskingManagerOrgName\":\"${<< parameters.org_name >>}\", \"TaskingManagerOrgCode\":\"${<< parameters.org_code >>}\", \"SentryBackendDSN\":\"${<< parameters.sentry_backend_dsn >>}\", \"TaskingManagerEmailContactAddress\":\"${<< parameters.email_contact_address >>}\"}'" >> $BASH_ENV
     - run:
         name: Install modules
         command: |
@@ -354,6 +356,8 @@ jobs:
         type: env_var_name
       homepage_stats_api_url:
         type: env_var_name
+      sentry_frontend_dsn:
+        type: env_var_name
     working_directory: /home/circleci/tasking-manager
     docker:
     - image: circleci/python:3.7-buster-node
@@ -419,6 +423,7 @@ jobs:
           export TM_IMAGE_UPLOAD_API_URL=${<< parameters.upload_api_url >>}
           export TM_USER_STATS_API_URL=${<< parameters.user_stats_api_url >>}
           export TM_HOMEPAGE_STATS_API_URL=${<< parameters.homepage_stats_api_url >>}
+          export TM_SENTRY_FRONTEND_DSN=${<< parameters.sentry_frontend_dsn >>}
           cd ${CIRCLE_WORKING_DIRECTORY}/frontend/
           yarn
           CI=true GENERATE_SOURCEMAP=false yarn build
@@ -481,6 +486,7 @@ workflows:
         org_homepage_video_url: TM_HOMEPAGE_VIDEO_URL #=https://cdn.hotosm.org/tasking-manager/mapping.mp4
         upload_api_url: TM_IMAGE_UPLOAD_API_URL
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
+        sentry_backend_dsn: TM_SENTRY_BACKEND_DSN
     - frontend_deploy:
         name: staging
         filters:
@@ -530,6 +536,7 @@ workflows:
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
         user_stats_api_url: TM_USER_STATS_API_URL
         homepage_stats_api_url: TM_HOMEPAGE_STATS_API_URL
+        sentry_frontend_dsn: TM_SENTRY_FRONTEND_DSN
     - backend_deploy:
         name: teachosm
         filters:
@@ -577,6 +584,7 @@ workflows:
         org_homepage_video_url: TM_HOMEPAGE_VIDEO_URL
         upload_api_url: TM_IMAGE_UPLOAD_API_URL
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
+        sentry_backend_dsn: TM_SENTRY_BACKEND_DSN
     - frontend_deploy:
         name: teachosm
         filters:
@@ -626,6 +634,7 @@ workflows:
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
         user_stats_api_url: TM_USER_STATS_API_URL
         homepage_stats_api_url: TM_HOMEPAGE_STATS_API_URL
+        sentry_frontend_dsn: TM_SENTRY_FRONTEND_DSN
     - backend_deploy:
         name: assisted
         filters:
@@ -673,6 +682,7 @@ workflows:
         org_homepage_video_url: TM_HOMEPAGE_VIDEO_URL
         upload_api_url: TM_IMAGE_UPLOAD_API_URL
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
+        sentry_backend_dsn: TM_SENTRY_BACKEND_DSN
     - frontend_deploy:
         name: assisted
         filters:
@@ -722,6 +732,7 @@ workflows:
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
         user_stats_api_url: TM_USER_STATS_API_URL
         homepage_stats_api_url: TM_HOMEPAGE_STATS_API_URL
+        sentry_frontend_dsn: TM_SENTRY_FRONTEND_DSN
   backend-production:
     jobs:
     - backend_deploy:
@@ -769,6 +780,7 @@ workflows:
         org_homepage_video_url: TM_HOMEPAGE_VIDEO_URL
         upload_api_url: TM_IMAGE_UPLOAD_API_URL
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
+        sentry_backend_dsn: TM_SENTRY_BACKEND_DSN
   frontend-production:
     jobs:
     - frontend_deploy:
@@ -819,3 +831,4 @@ workflows:
         upload_api_key: TM_IMAGE_UPLOAD_API_KEY
         user_stats_api_url: TM_USER_STATS_API_URL
         homepage_stats_api_url: TM_HOMEPAGE_STATS_API_URL
+        sentry_frontend_dsn: TM_SENTRY_FRONTEND_DSN

backend/__init__.py

@@ -9,10 +9,22 @@
 from flask_restful import Api
 from flask_sqlalchemy import SQLAlchemy
 
-
 from backend.config import EnvironmentConfig
 
 
+def sentry_init():
+    """Initialize sentry.io event tracking"""
+    import sentry_sdk
+    from sentry_sdk.integrations.flask import FlaskIntegration
+
+    sentry_sdk.init(
+        dsn=EnvironmentConfig.SENTRY_BACKEND_DSN,
+        environment=EnvironmentConfig.ENVIRONMENT,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=0.1,
+    )
+
+
 def format_url(endpoint):
     parts = "/".join([i for i in endpoint.split("/") if i])
     return "/api/{}/{}/".format(EnvironmentConfig.API_VERSION, parts)
@@ -33,6 +45,9 @@ def create_app(env=None):
     Bootstrap function to initialise the Flask app and config
     :return: Initialised Flask app
     """
+    # If SENTRY_BACKEND_DSN is configured, init sentry_sdk tracking
+    if EnvironmentConfig.SENTRY_BACKEND_DSN:
+        sentry_init()
 
     app = Flask(
         __name__,

backend/api/comments/resources.py

@@ -54,7 +54,7 @@ def post(self, project_id):
         """
         authenticated_user_id = token_auth.current_user()
         if UserService.is_user_blocked(authenticated_user_id):
-            return "User is on read only mode", 403
+            return {"Error": "User is on read only mode."}, 403
 
         try:
             chat_dto = ChatMessageDTO(request.get_json())

backend/config.py

@@ -24,6 +24,7 @@ class EnvironmentConfig:
     API_VERSION = os.getenv("TM_APP_API_VERSION", "v2")
     ORG_CODE = os.getenv("TM_ORG_CODE", "")
     ORG_NAME = os.getenv("TM_ORG_NAME", "")
+    ENVIRONMENT = os.getenv("TM_ENVIRONMENT", "")
     # The default tag used in the OSM changeset comment
     DEFAULT_CHANGESET_COMMENT = os.getenv("TM_DEFAULT_CHANGESET_COMMENT", None)
 
@@ -112,3 +113,6 @@ class EnvironmentConfig:
     # Image upload Api
     IMAGE_UPLOAD_API_KEY = os.getenv("TM_IMAGE_UPLOAD_API_KEY", None)
     IMAGE_UPLOAD_API_URL = os.getenv("TM_IMAGE_UPLOAD_API_URL", None)
+
+    # Sentry backend DSN
+    SENTRY_BACKEND_DSN = os.getenv("TM_SENTRY_BACKEND_DSN", None)

backend/models/postgis/project.py

@@ -447,8 +447,16 @@ def update(self, project_dto: ProjectDTO):
         else:
             if self.custom_editor:
                 self.custom_editor.delete()
-
-        self.campaign = [Campaign.query.get(c.id) for c in project_dto.campaigns]
+        # handle campaign update
+        try:
+            new_ids = [c.id for c in project_dto.campaigns]
+            new_ids.sort()
+        except TypeError:
+            new_ids = []
+        current_ids = [c.id for c in self.campaign]
+        current_ids.sort()
+        if new_ids != current_ids:
+            self.campaign = Campaign.query.filter(Campaign.id.in_(new_ids)).all()
 
         if project_dto.mapping_permission:
             self.mapping_permission = MappingPermission[
@@ -460,10 +468,16 @@ def update(self, project_dto: ProjectDTO):
                 project_dto.validation_permission.upper()
             ].value
 
-        # Update Interests.
-        self.interests = []
-        if project_dto.interests:
-            self.interests = [Interest.query.get(i.id) for i in project_dto.interests]
+        # handle interests update
+        try:
+            new_ids = [c.id for c in project_dto.interests]
+            new_ids.sort()
+        except TypeError:
+            new_ids = []
+        current_ids = [c.id for c in self.interests]
+        current_ids.sort()
+        if new_ids != current_ids:
+            self.interests = Interest.query.filter(Interest.id.in_(new_ids)).all()
         # try to update country info if that information is not present
         if len(self.country) == 0:
             self.set_country_info()

backend/services/messaging/chat_service.py

@@ -5,7 +5,6 @@
 from backend.services.project_service import ProjectService
 from backend.services.project_admin_service import ProjectAdminService
 from backend.services.team_service import TeamService
-from backend.services.users.user_service import UserService
 from backend.models.postgis.statuses import TeamRoles
 from backend.models.postgis.project import ProjectStatus
 from backend import db
@@ -19,9 +18,6 @@ def post_message(
         """ Save message to DB and return latest chat"""
         current_app.logger.debug("Posting Chat Message")
 
-        if UserService.is_user_blocked(authenticated_user_id):
-            raise ValueError("User is on read only mode")
-
         project = ProjectService.get_project_by_id(project_id)
         is_allowed_user = True
         is_manager_permission = ProjectAdminService.is_user_action_permitted_on_project(
@@ -66,7 +62,7 @@ def post_message(
             )
             db.session.commit()
             # Ensure we return latest messages after post
-            return ProjectChat.get_messages(chat_dto.project_id, 1)
+            return ProjectChat.get_messages(chat_dto.project_id, 1, 5)
         else:
             raise ValueError("User not permitted to post Comment")
 

docs/contributing-code.md

@@ -10,7 +10,7 @@ All of the Tasking Manager development is going to happen in the [project reposi
 1. Write tests for all new backend features and use a tool (like coveralls.io) to measure test coverage.
 2. Consider writing tests when building new frontend.
 3. Stick to pep8 python style guide for the backend.
-4. Apply JSLint style guide rules for the frontend code.
+4. Apply ESLint and [prettier](https://prettier.io/) style guide rules for the frontend code.
 5. Export translatable strings with `make refresh-translatables` and include them in your commit.
 
 ## Code collaboration and version control
@@ -61,7 +61,9 @@ of the commit message can be left out.
 
 Before sending a PR, make sure you run the following commands and include the changes in your commit.
 
-* Code formatting: Format all backend code correctly by running [Black](https://pypi.org/project/black/): `black manage.py backend tests migrations` and all frontend code by running [prettier](https://prettier.io/): `yarn prettier` (inside the `frontend` directory).
+* Code formatting:
+  * Format all backend code by running [Black](https://pypi.org/project/black/): `black manage.py backend tests migrations`
+  * Format all frontend code with [prettier](https://prettier.io/) either by [configuring your editor](https://prettier.io/docs/en/editors.html) or by running `yarn prettier` inside the `frontend` directory.
 * Coding standards: Make sure you adhere to the coding standards eventually risen by [Flake8](http://flake8.pycqa.org/en/latest/): `flake8 manage.py backend tests migrations`
 * Prepare for translations: In case you have introduced new strings on the frontend, the translation source file must be updated this can be done via `make refresh-translatables` or `yarn build-locales` (inside the `frontend` directory).
 

example.env

@@ -1,21 +1,16 @@
 # Tasking Manager configuration file
-#
-
 
-#
 # Copy to `example.env` and adjust to make it work!
 #
 
-# The base url the application is reachable (required)
-# On a local environment, use 127.0.0.1 not localhost, otherwise the authentication will not work
-# For local builds, the API URL is the same as the Base App URL
-#
+# The TM_APP_BASE_URL defines the URL of the frontend and is used by the backend
+# to configure links on emails and some authentication callbacks.
+# The TM_APP_API_URL can be the same as the frontend or not. It depends on how you'll deploy it.
 TM_APP_BASE_URL=http://127.0.0.1:5000
 TM_APP_API_URL=$TM_APP_BASE_URL
 TM_APP_API_VERSION=v2
 
 # Information about the hosting organization
-#
 TM_ORG_NAME="My organisation"
 TM_ORG_CODE=MYO
 # Don't use http or https on the following var
@@ -35,6 +30,9 @@ OSM_REGISTER_URL=https://www.openstreetmap.org/user/new
 ID_EDITOR_URL=https://www.openstreetmap.org/edit?editor=id
 POTLATCH2_EDITOR_URL=https://www.openstreetmap.org/edit?editor=potlatch2
 
+# Matomo configuration. Optional, configure it in case you have a Matomo instance.
+# TM_MATOMO_ID="site_id"
+# TM_MATOMO_ENDPOINT="https://..."
 
 # Mapbox access key to display the maps (optional)
 #
@@ -143,5 +141,11 @@ TM_DEFAULT_LOCALE=en
 #
 # TM_FRONTEND_BASE_URL=http://127.0.0.1:3000
 
-# This sets a file size limit to project geometry import. Define it in bytes.
+# This sets a file size limit to allow when importing a project geometry from a file. Define it in bytes.
 # TM_IMPORT_MAX_FILESIZE=1000000
+# Defines the maximum area allowed to the Projects' AoI. Default is 5000. The unit is square kilometers.
+# TM_MAX_AOI_AREA=5000
+
+# Sentry.io DSN Config (optional)
+# TM_SENTRY_BACKEND_DSN=foo
+# TM_SENTRY_FRONTEND_DSN=foo

frontend/.env.expand

@@ -24,6 +24,7 @@ REACT_APP_HOMEPAGE_VIDEO_URL=$TM_HOMEPAGE_VIDEO_URL
 REACT_APP_MAPBOX_TOKEN=$TM_MAPBOX_TOKEN
 REACT_APP_ENABLE_SERVICEWORKER=$TM_ENABLE_SERVICEWORKER
 REACT_APP_MAX_FILESIZE=$TM_IMPORT_MAX_FILESIZE
+REACT_APP_MAX_AOI_AREA=$TM_MAX_AOI_AREA
 REACT_APP_USER_STATS_API_URL=$TM_USER_STATS_API_URL
 REACT_APP_HOMEPAGE_STATS_API_URL=$TM_HOMEPAGE_STATS_API_URL
 REACT_APP_OSM_CONSUMER_KEY=$TM_CONSUMER_KEY
@@ -33,3 +34,5 @@ REACT_APP_TM_ORG_NAME=$TM_ORG_NAME
 REACT_APP_OSM_REGISTER_URL=$OSM_REGISTER_URL
 REACT_APP_ID_EDITOR_URL=$ID_EDITOR_URL
 REACT_APP_POTLATCH2_EDITOR_URL=$POTLATCH2_EDITOR_URL
+REACT_APP_SENTRY_FRONTEND_DSN=$TM_SENTRY_FRONTEND_DSN
+REACT_APP_ENVIRONMENT=$TM_ENVIRONMENT

frontend/package.json

@@ -4,18 +4,20 @@
   "license": "BSD-2-Clause",
   "private": false,
   "dependencies": {
-    "@formatjs/intl-pluralrules": "^3.4.7",
-    "@formatjs/intl-relativetimeformat": "^7.2.7",
+    "@formatjs/intl-pluralrules": "^3.4.8",
+    "@formatjs/intl-relativetimeformat": "^7.2.8",
     "@formatjs/intl-utils": "^3.8.4",
     "@formatjs/macro": "^0.2.8",
     "@hotosm/id": "^2.18.5",
     "@hotosm/iso-countries-languages": "^1.0.2",
     "@lokibai/react-use-copy-clipboard": "^1.0.4",
-    "@mapbox/geo-viewport": "^0.4.0",
+    "@mapbox/geo-viewport": "^0.4.1",
     "@mapbox/mapbox-gl-draw": "^1.1.2",
     "@mapbox/mapbox-gl-language": "^0.10.1",
     "@mapbox/togeojson": "^0.16.0",
     "@reach/router": "^1.3.4",
+    "@sentry/react": "^5.25.0",
+    "@sentry/tracing": "^5.25.0",
     "@turf/area": "^6.0.1",
     "@turf/bbox": "^6.0.1",
     "@turf/bbox-polygon": "^6.0.1",
@@ -26,23 +28,24 @@
     "@webscopeio/react-textarea-autocomplete": "^4.7.2",
     "axios": "^0.20.0",
     "chart.js": "^2.9.3",
-    "dompurify": "^2.0.17",
+    "dompurify": "^2.1.1",
     "downshift-hooks": "^0.8.1",
     "final-form": "^4.20.1",
     "fromentries": "^1.2.1",
-    "humanize-duration": "^3.23.1",
+    "humanize-duration": "^3.24.0",
     "immutable": "^4.0.0-rc.12",
     "mapbox-gl": "^1.12.0",
     "mapbox-gl-draw-circle": "^1.1.1",
     "mapbox-gl-draw-rectangle-mode": "^1.0.4",
-    "marked": "^1.1.1",
+    "marked": "^1.2.0",
     "node-sass": "^4.14.1",
     "osmtogeojson": "^3.0.0-beta.3",
-    "query-string": "^6.13.2",
+    "query-string": "^6.13.5",
     "react": "^16.13.1",
     "react-calendar-heatmap": "^1.8.1",
     "react-chartjs-2": "^2.10.0",
     "react-click-outside": "^3.0.1",
+    "react-datepicker": "^3.2.2",
     "react-dom": "^16.13.1",
     "react-dropzone": "^11.2.0",
     "react-final-form": "^6.5.1",
@@ -92,9 +95,9 @@
   "devDependencies": {
     "@testing-library/jest-dom": "^5.11.4",
     "@testing-library/react": "^11.0.4",
-    "@testing-library/react-hooks": "^3.4.1",
+    "@testing-library/react-hooks": "^3.4.2",
     "combine-react-intl-messages": "^4.0.0",
-    "jest-canvas-mock": "^2.2.0",
+    "jest-canvas-mock": "^2.3.0",
     "msw": "^0.21.2",
     "prettier": "^2.1.2",
     "react-test-renderer": "^16.13.1"

frontend/public/index.html

@@ -53,8 +53,8 @@
       _paq.push(['trackPageView']);
       _paq.push(['enableLinkTracking']);   // Tracks downloads
       _paq.push(['trackVisibleContentImpressions']);  // Tracks content blocks
-      var site_id = %REACT_APP_MATOMO_ID%;
-      if (typeof site_id !== 'undefined') {
+      var site_id = "%REACT_APP_MATOMO_ID%";
+      if (site_id && "%REACT_APP_MATOMO_ENDPOINT%") {
         (function() {
           var u="%REACT_APP_MATOMO_ENDPOINT%";
           _paq.push(['setTrackerUrl', u+'piwik.php']);