Merge #3041 #3094

3041: refractor(session service) using $state instead of $location r=jniles a=jeremielodi

closes #3030

3094: fix: migrate to utf8mb4 r=jniles a=jniles

This commit migrates the database to UTF8MB4 in preparation for MySQL 8 which defaults to this.  It also produces better results for diverse character sets.

Closes #2989.

**Overview**
The motivation for this Pull Request is two-fold:
 1. Make sure no one had [break the server](https://medium.com/@adamhooper/in-mysql-never-use-utf8-use-utf8mb4-11761243e434) by putting valid characters that [cannot be represented in MySQL's UTF8 character set](https://stackoverflow.com/questions/30074492/what-is-the-difference-between-utf8mb4-and-utf8-charsets-in-mysql#30074553).
 2. Ensure we are set up for future releases of MySQL and future languages by migrating to UTF8MB4.  MySQL 8 will support this out of the box.

And additional benefit continues the work started by @mbayopanda in his work on Travis CI changes in #3061 to document and standardize the character sets and collations in our database.  Previously, we just relied on having "the right set up", with some of our Stored Procedures in the server's default encoding, functions called in the mysqljs's default encoding, etc...  Now, we define at creation time and connection time the character sets and collations we are working with.

**Notes**
I had to [skip one test](https://github.com/IMA-WorldHealth/bhima-2.X/blob/master/test/end-to-end/verificationLinks/verificationLinks.spec.js#L19).  It was the link between debtor groups and patients.  I'm pretty sure this is due to the fact that the order data was inserted in changed somehow.  Either way, the test is non-specific and should be re-written to reference the group by its name.



Co-authored-by: jeremielodi <[email protected]>
Co-authored-by: Jonathan Niles <[email protected]>

client/src/js/services/Session.js

@@ -2,7 +2,7 @@ angular.module('bhima.services')
   .service('SessionService', SessionService);
 
 SessionService.$inject = [
-  '$sessionStorage', '$http', '$location', 'util', '$rootScope', '$q',
+  '$sessionStorage', '$http', '$state', 'util', '$rootScope', '$q',
 ];
 
 /**
@@ -19,7 +19,7 @@ SessionService.$inject = [
  *
  * @constructor
  */
-function SessionService($sessionStorage, $http, $location, util, $rootScope, $q) {
+function SessionService($sessionStorage, $http, $state, util, $rootScope, $q) {
   const service = this;
 
   // set up the storage instance
@@ -66,7 +66,7 @@ function SessionService($sessionStorage, $http, $location, util, $rootScope, $q)
     load();
 
     // TODO - use $state
-    $location.url('/login');
+    $state.go('login');
   }
 
   /**
@@ -86,7 +86,7 @@ function SessionService($sessionStorage, $http, $location, util, $rootScope, $q)
         create(session.user, session.enterprise, session.project, session.paths);
 
         // navigate to the main page
-        $location.url('/');
+        $state.go('index');
 
         // notify login event
         $rootScope.$emit('session:login');
@@ -111,7 +111,7 @@ function SessionService($sessionStorage, $http, $location, util, $rootScope, $q)
         $rootScope.$emit('session:logout');
 
         // navigate to the main page
-        $location.url('/login');
+        $state.go('login');
       });
   }
 

server/lib/barcode.js

@@ -54,7 +54,7 @@ function reverseLookup(barcodeKey) {
 
   const query = `
     SELECT BUID(uuid) as uuid FROM ${documentDefinition.table}
-    WHERE BUID(uuid) LIKE '${partialUuid}%' COLLATE utf8_unicode_ci;
+    WHERE BUID(uuid) LIKE '${partialUuid}%';
   `;
 
   // search for full UUID

server/lib/db/index.js

@@ -3,13 +3,12 @@ const q = require('q');
 const mysql = require('mysql');
 const uuidParse = require('uuid-parse');
 const uuidv4 = require('uuid/v4');
-const Transaction = require('./transaction');
 const _ = require('lodash');
+const debug = require('debug')('db');
 
 const BadRequest = require('../errors/BadRequest');
 const NotFound = require('../errors/NotFound');
-
-const debug = require('debug')('db');
+const Transaction = require('./transaction');
 
 /**
  * @class DatabaseConnector
@@ -32,7 +31,11 @@ class DatabaseConnector {
       user     : process.env.DB_USER,
       password : process.env.DB_PASS,
       database : process.env.DB_NAME,
-      charset  : 'utf8_unicode_ci',
+
+      // NOTE(@jniles): the MySQL character set variable must be uppercase.  To
+      // see the full list of check out:
+      // https://github.com/mysqljs/mysql/blob/master/lib/protocol/constants/charsets.js
+      charset  : 'UTF8MB4_UNICODE_CI',
     };
 
     this.pool = mysql.createPool(params);
@@ -115,8 +118,8 @@ class DatabaseConnector {
   one(sql, params, id, entity = 'record') {
     return this.exec(sql, params)
       .then(rows => {
-        const errorMessage =
-          `Expected ${entity} to contain a single record with id ${id}, but ${rows.length} were found!`;
+        // eslint-disable-next-line max-len
+        const errorMessage = `Expected ${entity} to contain a single record with id ${id}, but ${rows.length} were found!`;
 
         if (rows.length < 1) {
           debug(`#one(): Found too few records!  Expected 1 but ${rows.length} found.`);

server/models/bhima.sql

@@ -2,9 +2,12 @@
 -- be overriden by individual enterprises however these are the defaults
 
 -- set variables
-SET names 'utf8';
-SET character_set_database = 'utf8';
-SET collation_database = 'utf8_unicode_ci';
+SET foreign_key_checks = 0;
+
+SET names 'utf8mb4';
+SET character_set_database = 'utf8mb4';
+SET collation_database = 'utf8mb4_unicode_ci';
+SET CHARACTER SET utf8mb4, CHARACTER_SET_CONNECTION = utf8mb4;
 
 -- units
 INSERT INTO unit VALUES

server/models/functions.sql

@@ -134,7 +134,7 @@ BEGIN
     SET accountType = 'expense';
   END IF;
 
-  SET accountTypeId = (SELECT id FROM account_type WHERE `type` = accountType COLLATE utf8_unicode_ci LIMIT 1);
+  SET accountTypeId = (SELECT id FROM account_type WHERE `type` = accountType LIMIT 1);
 
   RETURN accountTypeId;
 END

server/models/icd10.sql

@@ -28,7 +28,7 @@ CREATE TABLE `icd10` (
   `label` text NOT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `icd10_code` (`code`)
-) ENGINE=InnoDB AUTO_INCREMENT=16384 DEFAULT CHARSET=utf8mb4;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE = utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --

server/models/procedures.sql

@@ -2,6 +2,12 @@
   This file contains all the stored procedures used in bhima's database.  It
   should be loaded after functions.sql.
 */
+
+SET names 'utf8mb4';
+SET character_set_database = 'utf8mb4';
+SET collation_database = 'utf8mb4_unicode_ci';
+SET CHARACTER SET utf8mb4, CHARACTER_SET_CONNECTION = utf8mb4;
+
 DELIMITER $$
 
 /*

server/models/procedures/account.sql

@@ -28,31 +28,31 @@ BEGIN
   DECLARE IMPORT_DEFAULT_OHADA_ACCOUNT_OPTION TINYINT(1) DEFAULT 1;
 
   SET existAccount = (SELECT IF((SELECT COUNT(`number`) AS total FROM `account` WHERE `number` = accountNumber) > 0, 1, 0));
-  SET existAccountType = (SELECT IF((SELECT COUNT(*) AS total FROM `account_type` WHERE `type` = accountType COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET accountTypeId = (SELECT id FROM `account_type` WHERE `type` = accountType COLLATE utf8_unicode_ci LIMIT 1);
+  SET existAccountType = (SELECT IF((SELECT COUNT(*) AS total FROM `account_type` WHERE `type` = accountType) > 0, 1, 0));
+  SET accountTypeId = (SELECT id FROM `account_type` WHERE `type` = accountType LIMIT 1);
   SET existAccountParent = (SELECT IF((SELECT COUNT(*) AS total FROM `account` WHERE `number` = accountParent) > 0, 1, 0));
 
   SET accountLength = (SELECT CHAR_LENGTH(accountNumber));
 
-  /* 
+  /*
     Handle parent account for importing ohada list of accounts
     We assume that ohada main accounts are already loaded into the system
   */
   IF (existAccountParent = 1) THEN
     SET accountParentId = (SELECT id FROM `account` WHERE `number` = accountParent);
   END IF;
-
 
-  /* 
-    Create account if it doesn't exist 
+
+  /*
+    Create account if it doesn't exist
 
     if the account already exist skip because we are in a loop and
     we have to continue importing other accounts
   */
   IF (existAccount = 0 AND existAccountType = 1) THEN
     INSERT INTO `account` (`type_id`, `enterprise_id`, `number`, `label`, `parent`) VALUES (accountTypeId, enterpriseId, accountNumber, accountLabel, accountParentId);
 
-    /* 
+    /*
       Insert default accounts for a quick usage
 
       insert an child account if the option is default ohada and we have an account with four digit

server/models/procedures/inventory.sql

@@ -5,9 +5,8 @@
 
   This procedure import a new inventory into the system
   by creating one and performing a stock integration
-  if necessary
+  if necessary.
 */
-
 DROP PROCEDURE IF EXISTS ImportInventory;
 CREATE PROCEDURE ImportInventory (
   IN enterpriseId SMALLINT(5),
@@ -29,43 +28,45 @@ BEGIN
   DECLARE inventoryTypeId TINYINT(3);
   DECLARE inventoryUnitId SMALLINT(5);
 
-  SET existInventoryGroup = (SELECT IF((SELECT COUNT(`name`) AS total FROM `inventory_group` WHERE `name` = inventoryGroupName COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventory = (SELECT IF((SELECT COUNT(`text`) AS total FROM `inventory` WHERE `code` = inventoryCode COLLATE utf8_unicode_ci OR `text` = inventoryText COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventoryType = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_type` WHERE `text` = inventoryType COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventoryUnit = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_unit` WHERE `text` = inventoryUnit COLLATE utf8_unicode_ci) > 0, 1, 0));
+  SET existInventoryGroup = (SELECT IF((SELECT COUNT(`name`) AS total FROM `inventory_group` WHERE `name` = inventoryGroupName) > 0, 1, 0));
+  SET existInventory = (SELECT IF((SELECT COUNT(`text`) AS total FROM `inventory` WHERE `code` = inventoryCode OR `text` = inventoryText) > 0, 1, 0));
+  SET existInventoryType = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_type` WHERE `text` = inventoryType) > 0, 1, 0));
+  SET existInventoryUnit = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_unit` WHERE `text` = inventoryUnit) > 0, 1, 0));
 
   /* Create group if doesn't exist */
   IF (existInventoryGroup = 0) THEN
     SET randomCode = (SELECT ROUND(RAND() * 10000000));
     SET inventoryGroupUuid = HUID(UUID());
     INSERT INTO `inventory_group` (`uuid`, `name`, `code`) VALUES (inventoryGroupUuid, inventoryGroupName, randomCode);
-  ELSE 
-    SET inventoryGroupUuid = (SELECT `uuid` FROM `inventory_group` WHERE `name` = inventoryGroupName COLLATE utf8_unicode_ci LIMIT 1);
+  ELSE
+    SET inventoryGroupUuid = (SELECT `uuid` FROM `inventory_group` WHERE `name` = inventoryGroupName LIMIT 1);
   END IF;
 
   /* Create type if doesn't exist */
-  IF (existInventoryType = 0) THEN 
+  IF (existInventoryType = 0) THEN
     SET inventoryTypeId = (SELECT MAX(`id`) + 1 FROM `inventory_type`);
     INSERT INTO `inventory_type` (`id`, `text`) VALUES (inventoryTypeId, inventoryType);
-  ELSE 
-    SET inventoryTypeId = (SELECT `id` FROM `inventory_type` WHERE LOWER(`text`) COLLATE utf8_unicode_ci = LOWER(inventoryType) LIMIT 1);
+  ELSE
+    SET inventoryTypeId = (SELECT `id` FROM `inventory_type` WHERE LOWER(`text`) = LOWER(inventoryType) LIMIT 1);
   END IF;
 
   /* Create unit if doesn't exist */
   IF (existInventoryUnit = 0) THEN
     SET inventoryUnitId = (SELECT MAX(`id`) + 1 FROM `inventory_unit`);
     INSERT INTO `inventory_unit` (`id`, `abbr`, `text`) VALUES (inventoryUnitId, inventoryUnit, inventoryUnit);
-  ELSE 
-    SET inventoryUnitId = (SELECT `id` FROM `inventory_unit` WHERE LOWER(`text`) COLLATE utf8_unicode_ci = LOWER(inventoryUnit) LIMIT 1);
+  ELSE
+    SET inventoryUnitId = (SELECT `id` FROM `inventory_unit` WHERE LOWER(`text`) = LOWER(inventoryUnit) LIMIT 1);
   END IF;
 
-  /* 
-    Create inventory if it doesn't exist 
+  /*
+    Create inventory if it doesn't exist
 
-    if the inventory already exist skip because we are in a loop and
+    If the inventory already exists, skip because we are in a loop and
     we have to continue importing other inventories
   */
   IF (existInventory = 0) THEN
-    INSERT INTO `inventory` (`enterprise_id`, `uuid`, `code`, `text`, `price`, `group_uuid`, `type_id`, `unit_id`) VALUES (enterpriseId, HUID(UUID()), inventoryCode, inventoryText, inventoryUnitPrice, inventoryGroupUuid, inventoryTypeId, inventoryUnitId);
+    INSERT INTO `inventory` (`enterprise_id`, `uuid`, `code`, `text`, `price`, `group_uuid`, `type_id`, `unit_id`)
+    VALUES
+    (enterpriseId, HUID(UUID()), inventoryCode, inventoryText, inventoryUnitPrice, inventoryGroupUuid, inventoryTypeId, inventoryUnitId);
   END IF;
 END $$