From 7dbd98abff44b268dec1ee15257c51052617e639 Mon Sep 17 00:00:00 2001 From: Neil Rotstan Date: Fri, 16 Nov 2018 10:21:48 -0800 Subject: [PATCH] Allow expert validators to classify mapping issues > Note: this includes a new database migration * Show validators in expert mode an additional form allowing them to (optionally) classify the type and number of problems they discover with mapping work that is leading to invalidation of the task * Record identified mapping issues in a `task_mapping_issues` database table * Display noted mapping issues in the task history immediately below the invalidation where they were recorded * Allow mapping issues fixed by the validator on behalf of the mapper to be noted when marking a task as validated * Display in the task history when a task was validated with fixes, and show the identified list of issues addressed by the validator * Add a new page, accessible from the account-nav dropdown menu, on which project managers and admins can manage the available mapping-issue categories * Seed the mapping-issue categories with a couple of initial categories * Add server RESTful API endpoints for managing mapping-issue categories * Add new `mapping_issue_categories` and `task_mapping_issues` database tables in new migration * Add support on server for optional inclusion of noted mapping issues during task validation/invalidation --- ...apping-issue-categories-edit.controller.js | 133 ++++++++++ .../mapping-issue-categories-edit.html | 89 +++++++ .../mapping-issue-categories.controller.js | 48 ++++ .../mapping-issue-categories.html | 57 +++++ .../account-nav/account-nav.directive.js | 8 + .../components/account-nav/account-nav.html | 5 + client/app/project/project.controller.js | 90 ++++++- client/app/project/project.html | 54 +++- client/app/services/mapping_issue.service.js | 129 ++++++++++ client/app/taskingmanager.app.js | 12 + client/assets/styles/sass/_project.scss | 19 ++ client/assets/styles/sass/_tables.scss | 16 ++ migrations/versions/22e7d7e0fa02_.py | 61 +++++ server/__init__.py | 4 + server/api/mapping_issues_apis.py | 234 ++++++++++++++++++ server/models/dtos/mapping_dto.py | 2 + server/models/dtos/mapping_issues_dto.py | 24 ++ server/models/dtos/validator_dto.py | 9 + server/models/postgis/mapping_issues.py | 73 ++++++ server/models/postgis/task.py | 48 +++- server/services/mapping_issues_service.py | 47 ++++ server/services/validator_service.py | 20 +- 22 files changed, 1171 insertions(+), 11 deletions(-) create mode 100644 client/app/admin/mapping-issues/mapping-issue-categories-edit.controller.js create mode 100644 client/app/admin/mapping-issues/mapping-issue-categories-edit.html create mode 100644 client/app/admin/mapping-issues/mapping-issue-categories.controller.js create mode 100644 client/app/admin/mapping-issues/mapping-issue-categories.html create mode 100644 client/app/services/mapping_issue.service.js create mode 100644 migrations/versions/22e7d7e0fa02_.py create mode 100644 server/api/mapping_issues_apis.py create mode 100644 server/models/dtos/mapping_issues_dto.py create mode 100644 server/models/postgis/mapping_issues.py create mode 100644 server/services/mapping_issues_service.py diff --git a/client/app/admin/mapping-issues/mapping-issue-categories-edit.controller.js b/client/app/admin/mapping-issues/mapping-issue-categories-edit.controller.js new file mode 100644 index 0000000000..f566f00201 --- /dev/null +++ b/client/app/admin/mapping-issues/mapping-issue-categories-edit.controller.js @@ -0,0 +1,133 @@ +(function () { + + 'use strict'; + + /** + * Mapping-issue Categories controller which manages viewing, creating and + * editing categories + */ + angular + .module('taskingManager') + .controller('mappingIssueCategoriesEditController', + ['$routeParams', '$location', 'mappingIssueService', mappingIssueCategoriesEditController]); + + function mappingIssueCategoriesEditController($routeParams, $location, mappingIssueService) { + var vm = this; + + vm.category = {}; + vm.isNew = false; + vm.isCategoryFound = false; + vm.isSaveEditsSuccessful = true; + vm.isArchiveSuccessful = true; + vm.isDeleteSuccessful = true; + vm.isCreateNewSuccessful = true; + vm.id = 0; + + activate(); + + function activate() { + vm.id = $routeParams.id; + if (vm.id === 'new'){ + vm.isNew = true; + } + else { + vm.id = parseInt(vm.id, 10); + var resultsPromise = mappingIssueService.getMappingIssueCategories(true); + resultsPromise.then(function (data) { + // On success + vm.category = data.categories.find(function(category) { + return category.categoryId === vm.id; + }); + + vm.isCategoryFound = !!vm.category; + }, function(){ + // On error + vm.isCategoryFound = false; + }); + } + } + + /** + * Cancel editing the category by going to the list of categories + */ + vm.cancel = function(){ + $location.path('/admin/mapping-issues/categories'); + }; + + /** + * Save the edits made to the category + */ + vm.saveEdits = function(){ + vm.isSaveEditsSuccessful = true; + var resultsPromise = mappingIssueService.updateMappingIssueCategory(vm.category, vm.id); + resultsPromise.then(function () { + // On success + $location.path('/admin/mapping-issues/categories'); + }, function () { + // On error + vm.isSaveEditsSuccessful = false; + }); + }; + + /** + * Archive the category + */ + vm.archive = function(){ + vm.category.archived = true; + vm.updateArchivedFlag() + }; + + /** + * Unarchive the category + */ + vm.unarchive = function(){ + vm.category.archived = false; + vm.updateArchivedFlag() + }; + + /** + * Perform an update of the category's archived flag + */ + vm.updateArchivedFlag = function() { + vm.isArchiveSuccessful = true; + var resultsPromise = mappingIssueService.updateMappingIssueCategory(vm.category, vm.id); + resultsPromise.then(function () { + // On success + $location.path('/admin/mapping-issues/categories'); + }, function () { + // On error + vm.isArchiveSuccessful = false; + }); + }; + + /** + * Delete the category + */ + vm.delete = function(){ + vm.isDeleteSuccessful = true; + var resultsPromise = mappingIssueService.deleteMappingIssueCategory(vm.id); + resultsPromise.then(function () { + // On success + $location.path('/admin/mapping-issues/categories'); + }, function () { + // On error + vm.isDeleteSuccessful = false; + }); + }; + + /** + * Create a new category + */ + vm.createNewMappingIssueCategory = function(){ + vm.isCreateNewSuccessful = true; + var resultsPromise = mappingIssueService.createMappingIssueCategory(vm.category); + resultsPromise.then(function () { + // On success + $location.path('/admin/mapping-issues/categories'); + }, function () { + // On error + vm.isCreateNewSuccessful = false; + }); + }; + } +})(); diff --git a/client/app/admin/mapping-issues/mapping-issue-categories-edit.html b/client/app/admin/mapping-issues/mapping-issue-categories-edit.html new file mode 100644 index 0000000000..6957b04fed --- /dev/null +++ b/client/app/admin/mapping-issues/mapping-issue-categories-edit.html @@ -0,0 +1,89 @@ +
+
+
+
+

+ {{ 'Edit Mapping-issue Category' | translate }} +

+
+
+
+
+
+
+ + + +
+
+ +
+
    +
  • + + +
    • +
  • + + +
    • +
+
+ + +
+
+ + +
+
+
+
+

{{ 'No category found.' | translate }}

+ +
+
+

{{ 'Failed to save edits. Please try again.' | translate }}

+
+
+

{{ 'Failed to archive/unarchive category. Please try again.' | translate }}

+
+
+

{{ 'Failed to delete category. Note that categories cannot be deleted once in use, but must be archived instead' | translate }}

+
+
+

{{ 'Failed to create a new category. Please try again.' | translate }}

+
+
+
+
+
+
+
diff --git a/client/app/admin/mapping-issues/mapping-issue-categories.controller.js b/client/app/admin/mapping-issues/mapping-issue-categories.controller.js new file mode 100644 index 0000000000..5284f4dbe0 --- /dev/null +++ b/client/app/admin/mapping-issues/mapping-issue-categories.controller.js @@ -0,0 +1,48 @@ +(function () { + + 'use strict'; + + /** + * Mapping-issue Categories controller which manages viewing, creating and + * editing categories of mapping issues that can be flagged during validation. + */ + angular + .module('taskingManager') + .controller('mappingIssueCategoriesController', + ['$location', 'mappingIssueService', mappingIssueCategoriesController]); + + function mappingIssueCategoriesController($location, mappingIssueService) { + var vm = this; + + vm.includeArchived = false; + vm.issueCategories = []; + + loadCategories(); + + function loadCategories() { + var resultsPromise = mappingIssueService.getMappingIssueCategories(vm.includeArchived); + resultsPromise.then(function (data) { + // On success + vm.issueCategories = data.categories; + }, function(){ + // On error + }); + } + + /** + * Toggle whether to fetch categories that have been archived when fetching + * the issue categories + */ + vm.toggleIncudeArchived = function(){ + vm.includeArchived = !vm.includeArchived; + loadCategories(); + }; + + /** + * Create a new license + */ + vm.createNewMappingIssueCategory = function(){ + $location.path('/admin/mapping-issues/categories/edit/new'); + } + } +})(); diff --git a/client/app/admin/mapping-issues/mapping-issue-categories.html b/client/app/admin/mapping-issues/mapping-issue-categories.html new file mode 100644 index 0000000000..7426ad6096 --- /dev/null +++ b/client/app/admin/mapping-issues/mapping-issue-categories.html @@ -0,0 +1,57 @@ +
+
+
+
+
+

+ {{ 'Mapping-issue Categories' | translate }} +

+
+ +
+ +
+

+ {{ 'Broad categories for classifying mapping issues noted during validation' | translate }} +

+
+ + +
+
+
+
+
+
+ + + + + + + + + + + + + + + + + +
{{ 'Name' | translate }}{{ 'Description' | translate }}{{ 'Archived' | translate }}{{ 'Edit' | translate }}
{{ category.name }}{{ category.description }} + + + + {{ 'Edit' | translate }} + +
+
+
+
+
+
diff --git a/client/app/components/account-nav/account-nav.directive.js b/client/app/components/account-nav/account-nav.directive.js index 46370cb71b..50bfebe7a3 100644 --- a/client/app/components/account-nav/account-nav.directive.js +++ b/client/app/components/account-nav/account-nav.directive.js @@ -113,6 +113,14 @@ $location.path('admin/licenses'); }; + /** + * Navigate to the mapping-issue category management page + */ + vm.goToManageMappingIssueCategories = function () { + $location.path('admin/mapping-issues/categories'); + vm.showDropdown = false; + }; + /** * Navigate to the project dashboard page */ diff --git a/client/app/components/account-nav/account-nav.html b/client/app/components/account-nav/account-nav.html index 81e86b2e70..4f2d8fccdb 100644 --- a/client/app/components/account-nav/account-nav.html +++ b/client/app/components/account-nav/account-nav.html @@ -50,6 +50,11 @@ {{ 'Manage licenses' | translate }} +
  • + + {{ 'Manage issues' | translate }} + +
  • {{ 'Project dashboard' | translate }} diff --git a/client/app/project/project.controller.js b/client/app/project/project.controller.js index f9006cd598..9d23bc9499 100644 --- a/client/app/project/project.controller.js +++ b/client/app/project/project.controller.js @@ -8,11 +8,12 @@ */ angular .module('taskingManager') - .controller('projectController', ['$timeout', '$interval', '$scope', '$location', '$routeParams', '$window', '$q', 'moment', 'configService', 'mapService', 'projectService', 'styleService', 'taskService', 'geospatialService', 'editorService', 'authService', 'accountService', 'userService', 'licenseService', 'messageService', 'drawService', 'languageService', 'userPreferencesService', projectController]); + .controller('projectController', ['$timeout', '$interval', '$scope', '$location', '$routeParams', '$window', '$q', 'moment', 'configService', 'mapService', 'projectService', 'styleService', 'taskService', 'mappingIssueService', 'geospatialService', 'editorService', 'authService', 'accountService', 'userService', 'licenseService', 'messageService', 'drawService', 'languageService', 'userPreferencesService', projectController]); - function projectController($timeout, $interval, $scope, $location, $routeParams, $window, $q, moment, configService, mapService, projectService, styleService, taskService, geospatialService, editorService, authService, accountService, userService, licenseService, messageService, drawService, languageService, userPreferencesService) { + function projectController($timeout, $interval, $scope, $location, $routeParams, $window, $q, moment, configService, mapService, projectService, styleService, taskService, mappingIssueService, geospatialService, editorService, authService, accountService, userService, licenseService, messageService, drawService, languageService, userPreferencesService) { var vm = this; vm.id = 0; + vm.selectedIssueCategory = null; vm.loaded = false; vm.projectData = null; vm.taskVectorLayer = null; @@ -56,6 +57,9 @@ vm.lockTime = {}; vm.multiSelectedTasksData = []; vm.multiLockedTasks = []; + vm.mappingIssueCategories = []; + vm.availableMappingIssueCategories = []; + vm.mappingIssues = []; //editor vm.editorStartError = ''; @@ -108,6 +112,12 @@ mapService.addOverviewMap(); vm.map = mapService.getOSMMap(); vm.loaded = false; + + mappingIssueService.getMappingIssueCategories().then(function(data) { + vm.mappingIssueCategories = data.categories; + vm.availableMappingIssueCategories = vm.remainingMappingIssueCategories(); + }); + vm.id = $routeParams.id; vm.highlightHistory = $routeParams.history ? parseInt($routeParams.history, 10) : null; @@ -166,6 +176,76 @@ return vm.map.getSize()[1] * 0.3; } + /** + * Returns array of mapping issue categories for which no issues have + * been noted as of yet + */ + vm.remainingMappingIssueCategories = function() { + return vm.mappingIssueCategories.filter(function(category) { + return !vm.isMappingIssueCategoryInUse(category); + }); + }; + + /** + * Returns true if mapping issues have already been noted in the given + * issue category + */ + vm.isMappingIssueCategoryInUse = function(category) { + return !!vm.mappingIssues.find(function(issue) { + return issue.mappingIssueCategoryId === category.categoryId; + }); + }; + + /** + * Add the currently selected mapping issue to the list of noted issues + * with an initial count of 1 issue + */ + vm.addMappingIssue = function() { + if (!vm.selectedIssueCategory) { + return; + } + + vm.mappingIssues.push({ + mappingIssueCategoryId: vm.selectedIssueCategory.categoryId, + issue: vm.selectedIssueCategory.name, + count: 1, + }); + + vm.availableMappingIssueCategories = vm.remainingMappingIssueCategories(); + vm.selectedIssueCategory = null; + }; + + /** + * Removes the given issue from the array of current mapping issues + */ + vm.removeMappingIssue = function(issueToRemove) { + vm.mappingIssues = vm.mappingIssues.filter(function(issue) { + return issue.mappingIssueCategoryId !== issueToRemove.mappingIssueCategoryId; + }); + + // Now that we've freed up an additional issue category, update the + // available categories + vm.availableMappingIssueCategories = vm.remainingMappingIssueCategories(); + }; + + /** + * Determines if any mapping issues are currently set + */ + vm.hasMappingIssues = function () { + return !!vm.mappingIssues.find(function(issue) { + return issue.count > 0; + }); + }; + + /** + * Reset mapping issue properties + */ + vm.resetMappingIssues = function() { + vm.mappingIssues = []; + vm.selectedIssueCategory = null; + vm.availableMappingIssueCategories = vm.remainingMappingIssueCategories(); + }; + /** * convenience method to reset task data controller properties */ @@ -175,6 +255,7 @@ vm.multiSelectedTasksData = []; vm.multiLockedTasks = []; vm.lockedTasksForCurrentUser = []; + vm.resetMappingIssues(); }; vm.updatePreferedEditor = function () { @@ -880,6 +961,7 @@ vm.lockedTaskData = null; vm.multiSelectedTasksData = []; vm.multiLockedTasks = []; + vm.resetMappingIssues(); vm.resetErrors(); vm.resetStatusFlags(); vm.clearCurrentSelection(); @@ -909,6 +991,7 @@ vm.lockedTaskData = null; vm.multiSelectedTasksData = []; vm.multiLockedTasks = []; + vm.resetMappingIssues(); setUpSelectedTask(data); // TODO: This is a bit icky. Need to find something better. Maybe when roles are in place. // Need to make a decision on what tab to go to if user has clicked map but is not on mapping or validating @@ -1128,7 +1211,8 @@ var tasks = [{ comment: comment, status: status, - taskId: taskId + taskId: taskId, + validationIssues: vm.mappingIssues, }]; var unLockPromise = taskService.unLockTaskValidation(projectId, tasks); vm.comment = ''; diff --git a/client/app/project/project.html b/client/app/project/project.html index 37e55d38ec..c01f0a6d0d 100644 --- a/client/app/project/project.html +++ b/client/app/project/project.html @@ -822,6 +822,30 @@
    {{ 'Get started by choosing your editor of choice.' | translate }}
    + +
    +
    {{ 'Note Mapping Issues (optional)' | translate }}
    +
    + + +
    + +
    +
    + + + +
    +
    +
    +
    {{ 'Done editing? Leave a comment and select one of the options below that matches your editing status.' | translate }}
    @@ -968,7 +992,7 @@
    {{ 'History' | translate }}
    {{ 'Automatically unlocked for validation' | translate }} {{ 'Marked as bad imagery' | translate }} {{ 'Mapped' | translate }} - {{ 'Validated' | translate }} + {{ 'Validated' | translate }} {{ (item.issues ? '(with fixes)' : '') | translate }} {{ 'Invalidated' | translate }} {{ 'Split' | translate }} {{ 'Marked as ready' | translate }} @@ -985,6 +1009,34 @@
    {{ 'History' | translate }}
    + + +
    + + {{ 'Noted Mapping Issues' | translate }} +
    +
      +
    • + {{ issue.name }}: {{issue.count}} +
      • +
    + + + + +
    + + {{ 'Fixed Mapping Issues' | translate }} +
    +
      +
    • + {{ issue.name }}: {{issue.count}} +
      • +
    + +
    diff --git a/client/app/services/mapping_issue.service.js b/client/app/services/mapping_issue.service.js new file mode 100644 index 0000000000..5102fa270f --- /dev/null +++ b/client/app/services/mapping_issue.service.js @@ -0,0 +1,129 @@ +(function () { + 'use strict'; + /** + * @fileoverview This file provides a service for task-mapping issues. + */ + + angular + .module('taskingManager') + .service('mappingIssueService', ['$http', '$q','configService', 'authService', mappingIssueService]); + + function mappingIssueService($http, $q, configService, authService) { + + var service = { + getMappingIssueCategory: getMappingIssueCategory, + createMappingIssueCategory: createMappingIssueCategory, + deleteMappingIssueCategory: deleteMappingIssueCategory, + updateMappingIssueCategory: updateMappingIssueCategory, + getMappingIssueCategories: getMappingIssueCategories, + }; + + return service; + + /** + * Get the mapping-issue category for the ID + * @param id - mapping-issue category id + * @returns {*} + */ + function getMappingIssueCategory(id) { + // Returns a promise + return $http({ + method: 'GET', + url: configService.tmAPI + '/mapping-issue-category/' + id, + headers: authService.getAuthenticatedHeader() + }).then(function successCallback(response) { + // this callback will be called asynchronously + // when the response is available + return response.data; + }, function errorCallback() { + // called asynchronously if an error occurs + // or server returns response with an error status. + return $q.reject("error"); + }); + } + + /** + * Create a new mapping-issue category + * @param categoryData + * @returns {*|!jQuery.deferred|!jQuery.jqXHR|!jQuery.Promise} + */ + function createMappingIssueCategory(categoryData){ + // Returns a promise + return $http({ + method: 'POST', + url: configService.tmAPI + '/mapping-issue-category', + data: categoryData, + headers: authService.getAuthenticatedHeader() + }).then(function successCallback(response) { + // this callback will be called asynchronously + // when the response is available + return response.data; + }, function errorCallback() { + // called asynchronously if an error occurs + // or server returns response with an error status. + return $q.reject("error"); + }); + } + + /** + * Delete a mapping-issue category + * @param id + * @returns {*|!jQuery.deferred|!jQuery.jqXHR|!jQuery.Promise} + */ + function deleteMappingIssueCategory(id){ + // Returns a promise + return $http({ + method: 'DELETE', + url: configService.tmAPI + '/mapping-issue-category/' + id, + headers: authService.getAuthenticatedHeader() + }).then(function successCallback(response) { + // this callback will be called asynchronously + // when the response is available + return response.data; + }, function errorCallback() { + // called asynchronously if an error occurs + // or server returns response with an error status. + return $q.reject("error"); + }); + } + + function updateMappingIssueCategory(categoryData, id){ + // Returns a promise + return $http({ + method: 'PUT', + url: configService.tmAPI + '/mapping-issue-category/' + id, + data: categoryData, + headers: authService.getAuthenticatedHeader() + }).then(function successCallback(response) { + // this callback will be called asynchronously + // when the response is available + return response.data; + }, function errorCallback() { + // called asynchronously if an error occurs + // or server returns response with an error status. + return $q.reject("error"); + }); + } + + /** + * Get all existing mapping-issue categories + * @returns {*|!jQuery.Promise|!jQuery.deferred|!jQuery.jqXHR} + */ + function getMappingIssueCategories(includeArchived){ + // Returns a promise + return $http({ + method: 'GET', + url: configService.tmAPI + '/mapping-issue-categories' + + (includeArchived ? '?includeArchived=true' : ''), + }).then(function successCallback(response) { + // this callback will be called asynchronously + // when the response is available + return response.data; + }, function errorCallback() { + // called asynchronously if an error occurs + // or server returns response with an error status. + return $q.reject("error"); + }); + } + } +})(); diff --git a/client/app/taskingmanager.app.js b/client/app/taskingmanager.app.js index 5d8ba80a83..e3ec988e4b 100644 --- a/client/app/taskingmanager.app.js +++ b/client/app/taskingmanager.app.js @@ -160,6 +160,18 @@ title: 'Edit Licenses' }) + .when('/admin/mapping-issues/categories', { + templateUrl: 'app/admin/mapping-issues/mapping-issue-categories.html', + controller: 'mappingIssueCategoriesController', + controllerAs: 'mappingIssueCategoriesCtrl' + }) + + .when('/admin/mapping-issues/categories/edit/:id', { + templateUrl: 'app/admin/mapping-issues/mapping-issue-categories-edit.html', + controller: 'mappingIssueCategoriesEditController', + controllerAs: 'mappingIssueCategoriesEditCtrl' + }) + .when('/admin/dashboard', { templateUrl: 'app/admin/dashboard/dashboard.html', controller: 'dashboardController', diff --git a/client/assets/styles/sass/_project.scss b/client/assets/styles/sass/_project.scss index ca9103424b..b2bae418fd 100644 --- a/client/assets/styles/sass/_project.scss +++ b/client/assets/styles/sass/_project.scss @@ -172,6 +172,25 @@ font-size: smaller; } +.mapping-issues { + .form-field { + label { + display: inline-block; + width: 175px; + } + + input[type="number"] { + max-width: 4em; + text-align: right; + margin: 0 0.5em; + } + } + + &__add-mapping-issue { + margin-top: 1.5em; + } +} + .chat--individual { margin-bottom: 1.5em; } diff --git a/client/assets/styles/sass/_tables.scss b/client/assets/styles/sass/_tables.scss index 0ee32231e0..4d79373961 100644 --- a/client/assets/styles/sass/_tables.scss +++ b/client/assets/styles/sass/_tables.scss @@ -39,6 +39,22 @@ word-break: break-word; } + td.issue-summary { + color: $darker-red; + + .issue-summary__heading { + margin-bottom: 1em; + } + + .issue-summary__marker { + font-size: 18px; + } + + &--fixed { + color: inherit; + } + } + th:first-child, td:first-child { padding-left: 1rem; diff --git a/migrations/versions/22e7d7e0fa02_.py b/migrations/versions/22e7d7e0fa02_.py new file mode 100644 index 0000000000..7057281e45 --- /dev/null +++ b/migrations/versions/22e7d7e0fa02_.py @@ -0,0 +1,61 @@ +"""empty message + +Revision ID: 22e7d7e0fa02 +Revises: fcd9cebaa79c +Create Date: 2018-08-06 15:31:03.973448 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '22e7d7e0fa02' +down_revision = 'fcd9cebaa79c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + categories_table = op.create_table( + 'mapping_issue_categories', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('archived', sa.Boolean(), nullable=False, server_default='false'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + + op.create_table('task_mapping_issues', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('mapping_issue_category_id', sa.Integer(), nullable=False), + sa.Column('task_history_id', sa.Integer(), nullable=False), + sa.Column('issue', sa.String(), nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['task_history_id'], ['task_history.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_mapping_issues_task_history_id'), 'task_mapping_issues', ['task_history_id'], unique=False) + op.create_foreign_key('fk_issue_category', 'task_mapping_issues', 'mapping_issue_categories', ['mapping_issue_category_id'], ['id']) + + # Setup some initial issue categories + initial_categories = [ + { 'id': 1, 'name': 'Missed Feature(s)' }, + { 'id': 2, 'name': 'Feature Geometry' }, + ] + + for category in initial_categories: + op.execute(categories_table.insert().values(name=category['name'])) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('fk_issue_category', 'task_mapping_issues', type_='foreignkey') + op.drop_index(op.f('ix_task_mapping_issues_task_history_id'), table_name='task_mapping_issues') + op.drop_table('task_mapping_issues') + op.drop_table('mapping_issue_categories') + # ### end Alembic commands ### diff --git a/server/__init__.py b/server/__init__.py index d91ea5ea5e..77944d1cf1 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -107,6 +107,7 @@ def init_flask_restful_routes(app): from server.api.swagger_docs_api import SwaggerDocsAPI from server.api.stats_api import StatsContributionsAPI, StatsActivityAPI, StatsProjectAPI, HomePageStatsAPI, StatsUserAPI from server.api.tags_apis import CampaignsTagsAPI, OrganisationTagsAPI + from server.api.mapping_issues_apis import MappingIssueCategoryAPI, MappingIssueCategoriesAPI from server.api.users.user_apis import UserAPI, UserIdAPI, UserOSMAPI, UserMappedProjects, UserSetRole, UserSetLevel,\ UserSetExpertMode, UserAcceptLicense, UserSearchFilterAPI, UserSearchAllAPI, UserUpdateAPI from server.api.validator_apis import LockTasksForValidationAPI, UnlockTasksAfterValidationAPI, StopValidatingAPI,\ @@ -166,6 +167,9 @@ def init_flask_restful_routes(app): api.add_resource(HomePageStatsAPI, '/api/v1/stats/summary') api.add_resource(CampaignsTagsAPI, '/api/v1/tags/campaigns') api.add_resource(OrganisationTagsAPI, '/api/v1/tags/organisations') + api.add_resource(MappingIssueCategoryAPI, '/api/v1/mapping-issue-category', endpoint="create_mapping_issue_category", methods=['POST']) + api.add_resource(MappingIssueCategoryAPI, '/api/v1/mapping-issue-category/', methods=['GET', 'PUT', 'DELETE']) + api.add_resource(MappingIssueCategoriesAPI, '/api/v1/mapping-issue-categories') api.add_resource(UserSearchAllAPI, '/api/v1/user/search-all') api.add_resource(UserSearchFilterAPI, '/api/v1/user/search/filter/') api.add_resource(UserAPI, '/api/v1/user/') diff --git a/server/api/mapping_issues_apis.py b/server/api/mapping_issues_apis.py new file mode 100644 index 0000000000..a97d560223 --- /dev/null +++ b/server/api/mapping_issues_apis.py @@ -0,0 +1,234 @@ +from flask_restful import Resource, current_app, request +from schematics.exceptions import DataError + +from server.models.dtos.mapping_issues_dto import MappingIssueCategoryDTO +from server.models.postgis.utils import NotFound +from server.services.mapping_issues_service import MappingIssueCategoryService +from server.services.users.authentication_service import token_auth, tm + + +class MappingIssueCategoryAPI(Resource): + @tm.pm_only() + @token_auth.login_required + def post(self): + """ + Creates a new mapping-issue category + --- + tags: + - mapping issues + produces: + - application/json + parameters: + - in: header + name: Authorization + description: Base64 encoded session token + required: true + type: string + default: Token sessionTokenHere== + - in: body + name: body + required: true + description: JSON object for creating a new mapping-issue category + schema: + properties: + name: + type: string + required: true + description: + type: string + responses: + 200: + description: New mapping-issue category created + 400: + description: Invalid Request + 401: + description: Unauthorized - Invalid credentials + 500: + description: Internal Server Error + """ + try: + category_dto = MappingIssueCategoryDTO(request.get_json()) + category_dto.validate() + except DataError as e: + current_app.logger.error(f'Error validating request: {str(e)}') + return str(e), 400 + + try: + new_category_id = MappingIssueCategoryService.create_mapping_issue_category(category_dto) + return {"categoryId": new_category_id}, 200 + except Exception as e: + error_msg = f'Mapping-issue category POST - unhandled error: {str(e)}' + current_app.logger.critical(error_msg) + return {"error": error_msg}, 500 + + def get(self, category_id): + """ + Get specified mapping-issue category + --- + tags: + - mapping issues + produces: + - application/json + parameters: + - name: category_id + in: path + description: The unique mapping-issue category ID + required: true + type: integer + default: 1 + responses: + 200: + description: Mapping-issue category found + 404: + description: Mapping-issue category not found + 500: + description: Internal Server Error + """ + try: + category_dto = MappingIssueCategoryService.get_category_as_dto(category_id) + return category_dto.to_primitive(), 200 + except NotFound: + return {"Error": "Mapping-issue category Not Found"}, 404 + except Exception as e: + error_msg = f'Mapping-issue category PUT - unhandled error: {str(e)}' + current_app.logger.critical(error_msg) + return {"error": error_msg}, 500 + + @tm.pm_only() + @token_auth.login_required + def put(self, category_id): + """ + Update an existing mapping-issue category + --- + tags: + - mapping issues + produces: + - application/json + parameters: + - in: header + name: Authorization + description: Base64 encoded session token + required: true + type: string + default: Token sessionTokenHere== + - name: category_id + in: path + description: The unique mapping-issue category ID + required: true + type: integer + default: 1 + - in: body + name: body + required: true + description: JSON object for updating a mapping-issue category + schema: + properties: + name: + type: string + description: + type: string + responses: + 200: + description: Mapping-issue category updated + 400: + description: Invalid Request + 401: + description: Unauthorized - Invalid credentials + 500: + description: Internal Server Error + """ + try: + category_dto = MappingIssueCategoryDTO(request.get_json()) + category_dto.category_id = category_id + category_dto.validate() + except DataError as e: + current_app.logger.error(f'Error validating request: {str(e)}') + return str(e), 400 + + try: + updated_category = MappingIssueCategoryService.update_mapping_issue_category(category_dto) + return updated_category.to_primitive(), 200 + except NotFound: + return {"Error": "Mapping-issue category Not Found"}, 404 + except Exception as e: + error_msg = f'Mapping-issue category PUT - unhandled error: {str(e)}' + current_app.logger.critical(error_msg) + return {"error": error_msg}, 500 + + @tm.pm_only() + @token_auth.login_required + def delete(self, category_id): + """ + Delete the specified mapping-issue category. Note that categories can + be deleted only if they have never been associated with a task. To + instead archive a used category that is no longer needed, update the + category with its archived flag set to true. + --- + tags: + - mapping issues + produces: + - application/json + parameters: + - in: header + name: Authorization + description: Base64 encoded session token + required: true + type: string + default: Token sessionTokenHere== + - name: category_id + in: path + description: The unique mapping-issue category ID + required: true + type: integer + default: 1 + responses: + 200: + description: Mapping-issue category deleted + 401: + description: Unauthorized - Invalid credentials + 404: + description: Mapping-issue category not found + 500: + description: Internal Server Error + """ + try: + MappingIssueCategoryService.delete_mapping_issue_category(category_id) + return {"Success": "Mapping-issue category deleted"}, 200 + except NotFound: + return {"Error": "Mapping-issue category Not Found"}, 404 + except Exception as e: + error_msg = f'Mapping-issue category DELETE - unhandled error: {str(e)}' + current_app.logger.critical(error_msg) + return {"error": error_msg}, 500 + + +class MappingIssueCategoriesAPI(Resource): + + def get(self): + """ + Gets all mapping issue categories + --- + tags: + - mapping issues + produces: + - application/json + parameters: + - in: query + name: includeArchived + description: Optional filter to include archived categories + type: boolean + default: false + responses: + 200: + description: Mapping issue categories + 500: + description: Internal Server Error + """ + try: + include_archived = request.args.get('includeArchived') == 'true' + categories = MappingIssueCategoryService.get_all_mapping_issue_categories(include_archived) + return categories.to_primitive(), 200 + except Exception as e: + error_msg = f'User GET - unhandled error: {str(e)}' + current_app.logger.critical(error_msg) + return {"error": error_msg}, 500 diff --git a/server/models/dtos/mapping_dto.py b/server/models/dtos/mapping_dto.py index 09cb7c3541..a558530fda 100644 --- a/server/models/dtos/mapping_dto.py +++ b/server/models/dtos/mapping_dto.py @@ -3,6 +3,7 @@ from schematics.types import StringType, IntType, DateTimeType, BooleanType from schematics.types.compound import ListType, ModelType from server.models.postgis.statuses import TaskStatus +from server.models.dtos.mapping_issues_dto import TaskMappingIssueDTO def is_valid_mapped_status(value): @@ -53,6 +54,7 @@ class TaskHistoryDTO(Model): action_text = StringType(serialized_name='actionText') action_date = DateTimeType(serialized_name='actionDate') action_by = StringType(serialized_name='actionBy') + issues = ListType(ModelType(TaskMappingIssueDTO)) class TaskDTO(Model): diff --git a/server/models/dtos/mapping_issues_dto.py b/server/models/dtos/mapping_issues_dto.py new file mode 100644 index 0000000000..41d843904d --- /dev/null +++ b/server/models/dtos/mapping_issues_dto.py @@ -0,0 +1,24 @@ +from schematics import Model +from schematics.types import IntType, StringType, BooleanType, ModelType +from schematics.types.compound import ListType + +class MappingIssueCategoryDTO(Model): + """ DTO used to define a mapping-issue category """ + category_id = IntType(serialized_name='categoryId') + name = StringType(required=True) + description = StringType(required=False) + archived = BooleanType(required=False) + +class MappingIssueCategoriesDTO(Model): + """ DTO for all mapping-issue categories """ + def __init__(self): + super().__init__() + self.categories = [] + + categories = ListType(ModelType(MappingIssueCategoryDTO)) + +class TaskMappingIssueDTO(Model): + """ DTO used to define a single mapping issue recorded with a task invalidation """ + category_id = IntType(serialized_name='categoryId') + name = StringType(required=True) + count = IntType(required=True) diff --git a/server/models/dtos/validator_dto.py b/server/models/dtos/validator_dto.py index 263a495d01..6abdc99153 100644 --- a/server/models/dtos/validator_dto.py +++ b/server/models/dtos/validator_dto.py @@ -26,17 +26,26 @@ class LockForValidationDTO(Model): preferred_locale = StringType(default='en') +class ValidationMappingIssue(Model): + """ Describes one or more occurences of an identified mapping problem during validation """ + mapping_issue_category_id = IntType(required=True, serialized_name='mappingIssueCategoryId') + issue = StringType(required=True) + count = IntType(required=True) + + class ValidatedTask(Model): """ Describes the model used to update the status of one task after validation """ task_id = IntType(required=True, serialized_name='taskId') status = StringType(required=True, validators=[is_valid_validated_status]) comment = StringType() + issues = ListType(ModelType(ValidationMappingIssue), serialized_name='validationIssues') class ResetValidatingTask(Model): """ Describes the model used to stop validating and reset the status of one task """ task_id = IntType(required=True, serialized_name='taskId') comment = StringType() + issues = ListType(ModelType(ValidationMappingIssue), serialized_name='validationIssues') class UnlockAfterValidationDTO(Model): diff --git a/server/models/postgis/mapping_issues.py b/server/models/postgis/mapping_issues.py new file mode 100644 index 0000000000..0aeac5b964 --- /dev/null +++ b/server/models/postgis/mapping_issues.py @@ -0,0 +1,73 @@ +from server import db +from server.models.dtos.mapping_issues_dto import MappingIssueCategoryDTO, MappingIssueCategoriesDTO + +class MappingIssueCategory(db.Model): + """ Represents a category of task mapping issues identified during validaton """ + __tablename__ = "mapping_issue_categories" + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String, nullable=False, unique=True) + description = db.Column(db.String, nullable=True) + archived = db.Column(db.Boolean, default=False, nullable=False) + + def __init__(self, name): + self.name = name + + @staticmethod + def get_by_id(category_id: int): + """ Get category by id """ + return MappingIssueCategory.query.get(category_id) + + @classmethod + def create_from_dto(cls, dto: MappingIssueCategoryDTO) -> int: + """ Creates a new MappingIssueCategory class from dto """ + new_category = cls(dto.name) + new_category.description = dto.description + + db.session.add(new_category) + db.session.commit() + + return new_category.id + + def update_category(self, dto: MappingIssueCategoryDTO): + """ Update existing category """ + self.name = dto.name + self.description = dto.description + if dto.archived is not None: + self.archived = dto.archived + db.session.commit() + + def delete(self): + """ Deletes the current model from the DB """ + db.session.delete(self) + db.session.commit() + + @staticmethod + def get_all_categories(include_archived): + category_query = MappingIssueCategory.query.order_by(MappingIssueCategory.name) + if not include_archived: + category_query = category_query.filter_by(archived=False) + + results = category_query.all() + if len(results) == 0: + raise NotFound() + + dto = MappingIssueCategoriesDTO() + for result in results: + category = MappingIssueCategoryDTO() + category.category_id = result.id + category.name = result.name + category.description = result.description + category.archived = result.archived + dto.categories.append(category) + + return dto + + def as_dto(self) -> MappingIssueCategoryDTO: + """ Convert the category to its DTO representation """ + dto = MappingIssueCategoryDTO() + dto.category_id = self.id + dto.name = self.name + dto.description = self.description + dto.archived = self.archived + + return dto diff --git a/server/models/postgis/task.py b/server/models/postgis/task.py index 35386554b8..4207a05268 100644 --- a/server/models/postgis/task.py +++ b/server/models/postgis/task.py @@ -12,6 +12,7 @@ from server.models.dtos.mapping_dto import TaskDTO, TaskHistoryDTO from server.models.dtos.validator_dto import MappedTasksByUser, MappedTasks, InvalidatedTask, InvalidatedTasks from server.models.dtos.project_dto import ProjectComment, ProjectCommentsDTO +from server.models.dtos.mapping_issues_dto import TaskMappingIssueDTO from server.models.postgis.statuses import TaskStatus, MappingLevel from server.models.postgis.user import User from server.models.postgis.utils import InvalidData, InvalidGeoJson, ST_GeomFromGeoJSON, ST_SetSRID, timestamp, parse_duration, NotFound @@ -103,6 +104,37 @@ def record_validation(project_id, task_id, validator_id, history): entry.updated_date = timestamp() +class TaskMappingIssue(db.Model): + """ Describes an issue (along with an occurrence count) with a task mapping that contributed to invalidation of the task """ + __tablename__ = "task_mapping_issues" + id = db.Column(db.Integer, primary_key=True) + task_history_id = db.Column(db.Integer, db.ForeignKey('task_history.id'), nullable=False, index=True) + issue = db.Column(db.String, nullable=False) + mapping_issue_category_id = db.Column(db.Integer, db.ForeignKey('mapping_issue_categories.id', name='fk_issue_category'), nullable=False) + count = db.Column(db.Integer, nullable=False) + + def __init__(self, issue, count, mapping_issue_category_id, task_history_id=None): + self.task_history_id = task_history_id + self.issue = issue + self.count = count + self.mapping_issue_category_id = mapping_issue_category_id + + def delete(self): + """ Deletes the current model from the DB """ + db.session.delete(self) + db.session.commit() + + def as_dto(self): + issue_dto = TaskMappingIssueDTO() + issue_dto.category_id = self.mapping_issue_category_id + issue_dto.name = self.issue + issue_dto.count = self.count + return issue_dto + + def __repr__(self): + return "{0}: {1}".format(self.issue, self.count) + + class TaskHistory(db.Model): """ Describes the history associated with a task """ __tablename__ = "task_history" @@ -117,6 +149,7 @@ class TaskHistory(db.Model): invalidation_history = db.relationship(TaskInvalidationHistory, lazy='dynamic', cascade='all') actioned_by = db.relationship(User) + task_mapping_issues = db.relationship(TaskMappingIssue, cascade="all") __table_args__ = (db.ForeignKeyConstraint([task_id, project_id], ['tasks.id', 'tasks.project_id'], name='fk_tasks'), db.Index('idx_task_history_composite', 'task_id', 'project_id'), {}) @@ -450,7 +483,7 @@ def is_mappable(self): return True - def set_task_history(self, action, user_id, comment=None, new_state=None): + def set_task_history(self, action, user_id, comment=None, new_state=None, mapping_issues=None): """ Sets the task history for the action that the user has just performed :param task: Task in scope @@ -458,6 +491,7 @@ def set_task_history(self, action, user_id, comment=None, new_state=None): :param action: Action the user has performed :param comment: Comment user has added :param new_state: New state of the task + :param mapping_issues: Identified issues leading to invalidation """ history = TaskHistory(self.id, self.project_id, user_id) @@ -470,6 +504,9 @@ def set_task_history(self, action, user_id, comment=None, new_state=None): elif action in [TaskAction.AUTO_UNLOCKED_FOR_MAPPING, TaskAction.AUTO_UNLOCKED_FOR_VALIDATION]: history.set_auto_unlock_action(action) + if mapping_issues is not None: + history.task_mapping_issues = mapping_issues + self.task_history.append(history) return history @@ -522,12 +559,13 @@ def record_auto_unlock(self, lock_duration): auto_unlocked.action_text = lock_duration self.update() - def unlock_task(self, user_id, new_state=None, comment=None, undo=False): + def unlock_task(self, user_id, new_state=None, comment=None, undo=False, issues=None): """ Unlock task and ensure duration task locked is saved in History """ if comment: - self.set_task_history(action=TaskAction.COMMENT, comment=comment, user_id=user_id) + self.set_task_history(action=TaskAction.COMMENT, comment=comment, user_id=user_id, mapping_issues=issues) - history = self.set_task_history(action=TaskAction.STATE_CHANGE, new_state=new_state, user_id=user_id) + history = self.set_task_history(action=TaskAction.STATE_CHANGE, new_state=new_state, + user_id=user_id, mapping_issues=issues) if new_state in [TaskStatus.MAPPED, TaskStatus.BADIMAGERY] and TaskStatus(self.task_status) != TaskStatus.LOCKED_FOR_VALIDATION: # Don't set mapped if state being set back to mapped after validation @@ -663,6 +701,8 @@ def as_dto_with_instructions(self, preferred_locale: str = 'en') -> TaskDTO: history.action_text = action.action_text history.action_date = action.action_date history.action_by = action.actioned_by.username if action.actioned_by else None + if action.task_mapping_issues: + history.issues = [issue.as_dto() for issue in action.task_mapping_issues] task_history.append(history) diff --git a/server/services/mapping_issues_service.py b/server/services/mapping_issues_service.py new file mode 100644 index 0000000000..b53f546ad5 --- /dev/null +++ b/server/services/mapping_issues_service.py @@ -0,0 +1,47 @@ +from server.models.postgis.mapping_issues import MappingIssueCategory +from server.models.dtos.mapping_issues_dto import MappingIssueCategoryDTO + +class MappingIssueCategoryService: + + @staticmethod + def get_mapping_issue_category(category_id: int) -> MappingIssueCategory: + """ + Get MappingIssueCategory from DB + :raises: NotFound + """ + category = MappingIssueCategory.get_by_id(category_id) + + if category is None: + raise NotFound() + + return category + + @staticmethod + def get_mapping_issue_category_as_dto(category_id: int) -> MappingIssueCategoryDTO: + """ Get MappingIssueCategory from DB """ + category = MappingIssueCategoryService.get_mapping_issue_category(category_id) + return category.as_dto() + + @staticmethod + def create_mapping_issue_category(category_dto: MappingIssueCategoryDTO) -> int: + """ Create MappingIssueCategory in DB """ + new_mapping_issue_category_id = MappingIssueCategory.create_from_dto(category_dto) + return new_mapping_issue_category_id + + @staticmethod + def update_mapping_issue_category(category_dto: MappingIssueCategoryDTO) -> MappingIssueCategoryDTO: + """ Create MappingIssueCategory in DB """ + category = MappingIssueCategoryService.get_mapping_issue_category(category_dto.category_id) + category.update_category(category_dto) + return category.as_dto() + + @staticmethod + def delete_mapping_issue_category(category_id: int): + """ Delete specified license""" + category = MappingIssueCategoryService.get_mapping_issue_category(category_id) + category.delete() + + @staticmethod + def get_all_mapping_issue_categories(include_archived): + """ Get all mapping issue categories""" + return MappingIssueCategory.get_all_categories(include_archived) diff --git a/server/services/validator_service.py b/server/services/validator_service.py index 6f4bdc8d59..2bcc5be0a5 100644 --- a/server/services/validator_service.py +++ b/server/services/validator_service.py @@ -4,7 +4,7 @@ from server.models.dtos.stats_dto import Pagination from server.models.dtos.validator_dto import LockForValidationDTO, UnlockAfterValidationDTO, MappedTasks, StopValidationDTO, InvalidatedTask, InvalidatedTasks from server.models.postgis.statuses import ValidatingNotAllowed -from server.models.postgis.task import Task, TaskStatus, TaskHistory, TaskInvalidationHistory +from server.models.postgis.task import Task, TaskStatus, TaskHistory, TaskInvalidationHistory, TaskAction, TaskMappingIssue from server.models.postgis.utils import NotFound, UserLicenseError, timestamp from server.models.postgis.project_info import ProjectInfo from server.services.messaging.message_service import MessageService @@ -117,7 +117,8 @@ def unlock_tasks_after_validation(validated_dto: UnlockAfterValidationDTO) -> Ta StatsService.update_stats_after_task_state_change(validated_dto.project_id, validated_dto.user_id, task_to_unlock['new_state'], task.id) - task.unlock_task(validated_dto.user_id, task_to_unlock['new_state'], task_to_unlock['comment']) + task_mapping_issues = ValidatorService.get_task_mapping_issues(task_to_unlock) + task.unlock_task(validated_dto.user_id, task_to_unlock['new_state'], task_to_unlock['comment'], issues=task_mapping_issues) dtos.append(task.as_dto_with_instructions(validated_dto.preferred_locale)) @@ -187,7 +188,8 @@ def get_tasks_locked_by_user(project_id: int, unlock_tasks, user_id: int): new_status = None tasks_to_unlock.append(dict(task=task, new_state=new_status, - comment=unlock_task.comment)) + comment=unlock_task.comment, + issues=unlock_task.issues)) return tasks_to_unlock @@ -271,3 +273,15 @@ def validate_all_tasks(project_id: int, user_id: int): project.tasks_mapped = (project.total_tasks - project.tasks_bad_imagery) project.tasks_validated = project.total_tasks project.save() + + @staticmethod + def get_task_mapping_issues(task_to_unlock: dict): + if task_to_unlock['issues'] is None: + return None + + # map ValidationMappingIssue DTOs to TaskMappingIssue instances for any issues + # that have count above zero. + return list(map(lambda issue_dto: TaskMappingIssue(issue=issue_dto.issue, + count=issue_dto.count, + mapping_issue_category_id=issue_dto.mapping_issue_category_id), + filter(lambda issue_dto: issue_dto.count > 0, task_to_unlock['issues'])))