Merge pull request #239 from madgik/dev_release

Dev release

.gitignore

@@ -6,9 +6,6 @@
 ## .pyc files
 *.pyc
 
-# Python
-__init__.py
-
 ## Directory-based project format:
 .idea/
 # if you remove the above rule, at least ignore the following:

Exareme-Docker/Dockerfile

@@ -71,13 +71,15 @@ RUN apk add --update curl ca-certificates && \
 # Some extra python libraries for the mip-algorithms, which needs to be
 # compiled by hand
 ADD files/requirements.txt /root/requirements.txt
-RUN apk add --update py-psycopg2 py-pip ca-certificates gcc musl-dev python-dev lapack-dev g++ gfortran && \
-    pip install -r /root/requirements.txt && \
-    pip install scipy==1.2.1 scikit-learn==0.20.3 && \
-    pip install pandas && \
-    pip install liac-arff && \
-    apk del py-pip ca-certificates gcc musl-dev python-dev lapack-dev gfortran && \
-    rm -rf /tmp/* /var/cache/apk/*
+RUN apk add --update py-psycopg2 py-pip ca-certificates gcc musl-dev python-dev lapack-dev g++ gfortran
+RUN pip install -r /root/requirements.txt
+RUN pip install scipy==1.2.1 scikit-learn==0.20.3
+RUN pip install pandas
+RUN apk add --update freetype-dev libjpeg-turbo-dev libpng-dev
+RUN pip install lifelines
+RUN pip install liac-arff
+RUN apk del py-pip ca-certificates gcc musl-dev python-dev lapack-dev gfortran
+RUN rm -rf /tmp/* /var/cache/apk/*
 
 # Runtime dependencies for Exareme
 RUN apk add --update rsync curl bash jq python py-requests lapack --no-cache procps && \
@@ -102,6 +104,7 @@ EXPOSE 9090
 EXPOSE 22
 
 ENV USER=root
+ENV PYTHONPATH "${PYTONPATH}:/root/mip-algorithms"
 WORKDIR /root/exareme
 
 CMD ["/bin/bash","bootstrap.sh"]

Exareme-Docker/files/requirements.txt

@@ -1,4 +1,4 @@
 apsw==3.9.2-r1
 NumPy==1.16.3
 titus
-
+patsy

Exareme-Docker/files/root/exareme/bootstrap.sh

@@ -1,22 +1,17 @@
 #!/usr/bin/env bash
 
-if [[ -z "${EXAREME_ACTIVE_WORKERS_PATH}" ]]; then
-	echo "Env. variable 'active_workers' not initialized in docker-compose.yaml files. Exiting..."
-	exit 1
-fi
+#Init environmental variables
 
-if [[ -z "${EXAREME_MASTER_PATH}" ]]; then
-	echo "Env. variable 'master' not initialized in docker-compose.yaml files. Exiting..."
-	exit 1
-fi
+export DOCKER_DATA_FOLDER="/root/exareme/data/"
+export DOCKER_METADATA_FOLDER="/root/exareme/data/"
+export EXAREME_ACTIVE_WORKERS_PATH="active_workers"
+export EXAREME_MASTER_PATH="master"
+export DATA="data"
 
-if [[ -z "${DATA}" ]]; then
-	echo "Env. variable 'data' not initialized in docker-compose.yaml files. Exiting..."
-	exit 1
-fi
-if [[ -z ${CONSULURL} ]]; then echo "CONSULURL is unset"; exit; fi
-if [[ -z ${NODE_NAME} ]]; then echo "NODE_NAME is unset";exit;  fi
-if [[ -z ${DOCKER_DATA_FOLDER} ]]; then echo "DOCKER_DATA_FOLDER is unset"; exit; fi
+if [[ -z ${CONSULURL} ]]; then echo "CONSULURL is unset. Check docker-compose file."; exit; fi
+if [[ -z ${NODE_NAME} ]]; then echo "NODE_NAME is unset. Check docker-compose file.";exit;  fi
+if [[ -z ${FEDERATION_ROLE} ]]; then echo "FEDERATION_ROLE is unset. Check docker-compose file.";exit;  fi
+if [[ -z ${ENVIRONMENT_TYPE} ]]; then echo "ENVIRONMENT_TYPE is unset. Check docker-compose file.";exit;  fi
 
 #Stop Exareme service
 stop_exareme () {
@@ -46,7 +41,7 @@ transformCsvToDB () {
 	# Removing all previous .db files from the DOCKER_DATA_FOLDER
 	echo "Deleting previous db files. "
 	rm -rf ${DOCKER_DATA_FOLDER}/**/*.db
-	
+
 	echo "Parsing the csv files in " ${DOCKER_DATA_FOLDER} " to db files. "
 	python ./convert-csv-dataset-to-db.py -f ${DOCKER_DATA_FOLDER} -t ${1}
 	#Get the status code from previous command
@@ -64,7 +59,7 @@ trap term_handler SIGTERM SIGKILL
 #This funciton will be executed when the container receives the SIGTERM signal (when stopping)
 term_handler () {
 
-if [[ "${MASTER_FLAG}" != "master" ]]; then   #worker
+if [[ "${FEDERATION_ROLE}" != "master" ]]; then   #worker
 	echo "*******************************Stopping Worker**************************************"
 	if [[ "$(curl -s ${CONSULURL}/v1/health/state/passing | jq -r '.[].Status')" = "passing" ]];  then
 		deleteKeysFromConsul "$EXAREME_ACTIVE_WORKERS_PATH"
@@ -89,7 +84,7 @@ exit 0
 mkdir -p  /tmp/demo/db/
 
 #This is the Worker
-if [[ "${MASTER_FLAG}" != "master" ]]; then
+if [[ "${FEDERATION_ROLE}" != "master" ]]; then
 
 	DESC="exareme-worker"
 	MY_IP=$(/sbin/ifconfig eth0 | grep "inet" | awk -F: '{print $2}' | cut -d ' ' -f 1)
@@ -148,21 +143,21 @@ if [[ "${MASTER_FLAG}" != "master" ]]; then
 		fi
 	done
 
-    if [[ ${TAG} == "dev" ]]; then
+    if [[ ${ENVIRONMENT_TYPE} == "DEV" ]] || [[ ${ENVIRONMENT_TYPE} == "TEST" ]]; then
         echo "Running set-local-datasets."
 	    ./set-local-datasets.sh
 
         echo -e "\nDEV version: Worker node["${MY_IP}","${NODE_NAME}"] may be connected to Master node["${MASTER_IP}","${MASTER_NAME}"]"
         curl -s -X PUT -d @- ${CONSULURL}/v1/kv/${EXAREME_ACTIVE_WORKERS_PATH}/${NODE_NAME} <<< ${MY_IP}
-    elif [[ ${TAG} == "prod" ]]; then
+    elif [[ ${ENVIRONMENT_TYPE} == "PROD" ]]; then
         #Health check for Worker. HEALTH_CHECK algorithm execution
         echo "Health check for Worker node["${MY_IP}","${NODE_NAME}"]"
 
         check="$(curl -s ${MASTER_IP}:9092/check/worker?IP_MASTER=${MASTER_IP}?IP_WORKER=${MY_IP})"
 
         if [[ -z ${check} ]]; then
             #If curl returned nothing, something is wrong. We can not know what is wrong though..
-            printf "Health_Check algorithm did not return anything...Switch TAG to 'dev' to see Error messages coming\
+            printf "Health_Check algorithm did not return anything...Switch ENVIRONMENT_TYPE to 'DEV' to see Error messages coming\
 from EXAREME..Exiting"
             exit 1
         else
@@ -187,7 +182,7 @@ from EXAREME..Exiting"
                 else
                     echo ${check}
                     echo "Worker node["${MY_IP}","${NODE_NAME}]" seems that is not connected with the Master..\
-Switch TAG to 'dev' to see Error messages coming from EXAREME..Exiting..."
+Switch ENVIRONMENT_TYPE to 'DEV' to see Error messages coming from EXAREME..Exiting..."
                     exit 1
                 fi
             fi
@@ -253,21 +248,21 @@ else
 			fi
 		done
 
-        if [[ ${TAG} == "dev" ]]; then
+        if [[ ${ENVIRONMENT_TYPE} == "DEV" ]] || [[ ${ENVIRONMENT_TYPE} == "TEST" ]] ; then
              echo "Running set-local-datasets."
 		    ./set-local-datasets.sh
 
             echo -e "\nDEV version: Master node["${MY_IP}","${NODE_NAME}"] may be initialized"
             curl -s -X PUT -d @- ${CONSULURL}/v1/kv/${EXAREME_MASTER_PATH}/${NODE_NAME} <<< ${MY_IP}
-        elif [[ ${TAG} == "prod" ]]; then
+        elif [[ ${ENVIRONMENT_TYPE} == "PROD" ]]; then
 		    #Health check for Master. HEALTH_CHECK algorithm execution
 		    echo "Health check for Master node["${MY_IP}","${NODE_NAME}"]"
 
 		    check=$(curl -s ${MY_IP}:9092/check/worker?IP_MASTER=${MY_IP}?IP_WORKER=${MY_IP})    #Master has a Worker instance. So in this case IP_MASTER / IP_WORKER is the same
 
             if [[ -z ${check} ]]; then
             #if curl returned nothing, something is wrong. We can not know what is wrong though
-                printf "Health_Check algorithm did not return anything...Switch TAG to 'dev' to see Error messages coming\
+                printf "Health_Check algorithm did not return anything...Switch ENVIRONMENT_TYPE to 'DEV' to see Error messages coming\
 from EXAREME..Exiting"
                 exit 1
             else
@@ -292,7 +287,7 @@ from EXAREME..Exiting"
                     else
                         echo ${check}
                         echo "Master node["${MY_IP}","${NODE_NAME}]" seems that could not be initialized..\
-Switch TAG to 'dev' to see Error messages coming from EXAREME..Exiting..."
+Switch ENVIRONMENT_TYPE to 'DEV' to see Error messages coming from EXAREME..Exiting..."
                         exit 1
                     fi
                 fi

Exareme-Docker/src/exareme/exareme-master/src/main/java/madgik/exareme/master/queryProcessor/composer/Composer.java

@@ -67,7 +67,7 @@ private static String createLocalTableQuery(AlgorithmProperties algorithmPropert
                 try {   // build query
                     SqlQueryResult sqlQueryResult = sqlBuilder.build(parameter.getValue());
                     filters = String.valueOf(sqlQueryResult);
-                    filters = filters.replaceAll("'", "\"");
+                    filters = filters.replace("'", "\"");
                     log.debug(filters);
                 } catch (IOException e) {
                     e.printStackTrace();
@@ -627,7 +627,7 @@ private static String composePythonLocalAlgorithmsDFLScript(
         dflScript.append("distributed create table " + outputGlobalTbl + " as direct \n");
         dflScript.append("select * from (\n  call_python_script 'python " + localPythonScriptPath + " ");
         for (ParameterProperties parameter : algorithmParameters) {
-            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replaceAll("\"", "\\\"")));
+            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replace("\"", "\\\"")));
         }
         dflScript.append(String.format("-%s \"%s\" ", ComposerConstants.inputLocalDBKey, inputLocalDB));
         dflScript.append(String.format("-%s \"%s\" ", ComposerConstants.dbQueryKey, dbQuery));
@@ -669,7 +669,7 @@ private static String composePythonLocalGlobalAlgorithmsDFLScript(
         dflScript.append("distributed create temporary table input_global_tbl to 1 as virtual \n");
         dflScript.append("select * from (\n  call_python_script 'python " + localPythonScriptPath + " ");
         for (ParameterProperties parameter : algorithmParameters) {
-            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replaceAll("\"", "\\\"")));
+            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replace("\"", "\\\"")));
         }
         dflScript.append(String.format("-%s \"%s\" ", ComposerConstants.inputLocalDBKey, inputLocalDB));
         dflScript.append(String.format("-%s \"%s\" ", ComposerConstants.dbQueryKey, dbQuery));
@@ -684,7 +684,7 @@ private static String composePythonLocalGlobalAlgorithmsDFLScript(
                         outputGlobalTbl));
         dflScript.append("select * from (\n  call_python_script 'python " + globalPythonScriptPath + " ");
         for (ParameterProperties parameter : algorithmParameters) {
-            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replaceAll("\"", "\\\"")));
+            dflScript.append(String.format("-%s \"%s\" ", parameter.getName(), parameter.getValue().replace("\"", "\\\"")));
         }
         dflScript.append(String.format("-%s \"%s\" ", ComposerConstants.localDBsKey, transferDBFilePath));
         dflScript.append(

Exareme-Docker/src/exareme/exareme-tools/madis/src/functionslocal/row/create_query.py

@@ -78,15 +78,23 @@ def create_complex_query(*args):
     if columnnamesA =='': columnnamesA = "null"
     mycolumnsA = columnnamesA.split(",");
 
-    if len(args)==6:
+    if len(args) >= 6:
         columnnamesB = args[5]
         mycolumnsB = columnnamesB.split(",");
         print mycolumnsB
+    if len(args) == 7:
+        columnnamesC = args[6]
+        mycolumnsC = columnnamesC.split(",");
+        print mycolumnsC
 
     query = init
 
     for i in xrange(len(mycolumnsA)):
-        if len(args)==6:
+        if len(args)==7:
+            istep=step.replace("???", mycolumnsC[i])
+            istep=istep.replace("??", mycolumnsB[i])
+            istep=istep.replace("?", mycolumnsA[i])
+        elif len(args)==6:
             istep=step.replace("??", mycolumnsB[i])
             istep=istep.replace("?", mycolumnsA[i])
         else:
@@ -120,4 +128,4 @@ def create_complex_query(*args):
         reload(sys)
         sys.setdefaultencoding('utf-8')
         import doctest
-        doctest.testmod()
+        doctest.testmod()

Exareme-Docker/src/exareme/exareme-tools/madis/src/functionslocal/row/errorchecking.py

@@ -37,7 +37,7 @@ def variableshouldbebinary_inputerrorchecking(nameofvariable, numberdistictvalue
     if numberdistictvalues == 2 :
         return "OK"
     else:
-        raise functions.OperatorError("ExaremeError", "Variable: " +nameofvariable+ " should be binary")
+        raise functions.OperatorError("ExaremeError", "Variable: " +nameofvariable+ " should have two distinct values. Here it gets "  + str(numberdistictvalues) +" distinct value(/s)")
 variableshouldbebinary_inputerrorchecking.registered = True