Freeze to master

Makefile

@@ -22,6 +22,7 @@ export API_TEST_JSON_PATH=swagger
 export PORT=8081
 export BACKEND_PORT=8081
 export TIMEOUT=30
+export BACKUP_TIMEOUT=1800
 # auth method - do not use auth by default (auth can be both passwords and OAuth)
 export NO_AUTH=True
 export TWITTER_OAUTH_ID=None
@@ -78,21 +79,25 @@ export MATCHID_DATA_BUCKET=$(shell echo ${APP_GROUP} | tr '[:upper:]' '[:lower:]
 export MATCHID_CONFIG_BUCKET=$(shell echo ${APP_GROUP} | tr '[:upper:]' '[:lower:]')
 
 # elasticsearch defaut configuration
+export ES_INDEX=${APP_GROUP}
 export ES_NODES = 1		# elasticsearch number of nodes
 export ES_SWARM_NODE_NUMBER = 2		# elasticsearch number of nodes
-export ES_MEM = 1024m		# elasticsearch : memory of each node
-export ES_VERSION = 7.16.3
+export ES_MEM := 1024m		# elasticsearch : memory of each node
+export ES_JAVA_OPTS := -Xms${ES_MEM} -Xmx${ES_MEM}		# elasticsearch : java options
+export ES_VERSION = 8.6.1
 export ES_DATA = ${BACKEND}/esdata
 export ES_THREADS = 2
 export ES_MAX_TRIES = 3
 export ES_CHUNK = 500
-export ES_BACKUP_FILE := $(shell echo esdata_`date +"%Y%m%d"`.tar)
-export ES_BACKUP_FILE_SNAR = esdata.snar
+export ES_BACKUP_NAME := $(shell echo esdata_`date +"%Y%m%d"`)
+export ES_BACKUP_FILE := ${ES_BACKUP_NAME}.tar
+export ES_BACKUP_FILE_SNAR = ${ES_BACKUP_NAME}.snar
 
 export DB_SERVICES=elasticsearch postgres
 
 export SERVICES=${DB_SERVICES} backend frontend
 
+-include ${APP_PATH}/${GIT_TOOLS}/artifacts.SCW
 dummy		    := $(shell touch artifacts)
 include ./artifacts
 
@@ -189,6 +194,106 @@ endif
 elasticsearch2-stop:
 	@${DC} -f ${DC_FILE}-elasticsearch-huge-remote.yml down
 
+elasticsearch-repository-plugin: elasticsearch-start
+	@if [ ! -f "elasticsearch-repository-plugin" ]; then\
+		echo installing elasticsearch repository plugin;\
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \
+			"echo ${STORAGE_ACCESS_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key";\
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch sh -c \
+			"echo ${STORAGE_SECRET_KEY} | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key";\
+		docker restart ${DC_PREFIX}-elasticsearch;\
+		timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0"  ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/ > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch API to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ;\
+		echo; touch elasticsearch-repository-plugin ; exit $$ret;\
+	fi;
+
+elasticsearch-repository-config: elasticsearch-repository-plugin
+	@if [ ! -f "elasticsearch-repository-config" ]; then\
+		echo creating elasticsearch repository ${APP_GROUP} in s3 bucket ${REPOSITORY_BUCKET} && \
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+			curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}" -H 'Content-Type: application/json' \
+			-d '{"type": "s3","settings": {"bucket": "${REPOSITORY_BUCKET}","client": "default","region": "${SCW_REGION}","endpoint": "${SCW_ENDPOINT}","path_style_access": true,"protocol": "https"}}' \
+			| grep -q '"acknowledged":true' && touch elasticsearch-repository-config;\
+	fi
+
+elasticsearch-freeze:
+	@\
+	docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+		curl -s -XPUT "localhost:9200/${ES_INDEX}/_settings" -H 'Content-Type: application/json' \
+			-d '{"index":{"blocks.write": true}}' | grep -q '"acknowledged":true' \
+	&& echo "index ${ES_INDEX} frozen" \
+	|| echo "index ${ES_INDEX} freeze failed"
+
+elasticsearch-repository-backup: elasticsearch-repository-config elasticsearch-freeze
+	@\
+	docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+		curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}" -H 'Content-Type: application/json'\
+			-d '{"indices": "${ES_INDEX}", "ignore_unavailable": true, "include_global_state": false}' \
+	| grep -q '{"accepted":true}';\
+	if [ "$$?" -ne "0" ]; then\
+		echo "snapshot ${ES_BACKUP_NAME} creation failed";\
+		exit 1;\
+	fi;\
+	echo -n creating snapshot ${ES_BACKUP_NAME} in elasticsearch repository;\
+	timeout=${BACKUP_TIMEOUT} ; ret=1 ; dot_count=0 ;\
+	until [ "$$timeout" -le 0 -o "$$ret" -eq "0"  ] ; do\
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+			curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}"\
+		| grep -q '"state":"SUCCESS"';\
+		ret=$$? ; \
+		if [ "$$ret" -ne "0" ] ; then\
+			echo -en "." ; \
+			((dot_count++));\
+			if [ "$$dot_count" -gt "10" ]; then\
+				echo -en "\rwaiting for snapshot ${ES_BACKUP_NAME} to complete $$timeout" ;\
+				dot_count=0;\
+			fi;\
+		fi ;\
+		((timeout--));((timeout--)); sleep 2 ; \
+	done ; echo ;\
+	if [ "$$ret" -ne "0" ]; then\
+		echo "snapshot ${ES_BACKUP_NAME} creation failed";\
+		exit $$ret;\
+	fi;\
+	echo "snapshot ${ES_BACKUP_NAME} created in elasticsearch repository" && touch elasticsearch-repository-backup
+
+
+elasticsearch-repository-backup-async: elasticsearch-repository-config
+	@docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+		curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}" -H 'Content-Type: application/json'\
+		-d '{"indices": "${ES_INDEX}", "ignore_unavailable": true, "include_global_state": false}'
+
+elasticsearch-repository-delete: elasticsearch-repository-config
+	@(\
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+			curl -s -XDELETE "localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}"\
+		> /dev/null 2>&1\
+	 ) && echo "snapshot ${ES_BACKUP_NAME} deleted from elasticsearch repository"
+
+elasticsearch-repository-list: elasticsearch-repository-config
+	@docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+			curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/_all"\
+		| jq -r '.snapshots[].snapshot'
+
+elasticsearch-repository-restore: elasticsearch-repository-config
+	@echo restoring snapshot ${ES_BACKUP_NAME} from elasticsearch repository;\
+	(\
+		docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+			curl -s -XPOST localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}/_restore?wait_for_completion=true -H 'Content-Type: application/json'\
+			-d '{"indices": "${ES_INDEX}","ignore_unavailable": true,"include_global_state": false}' \
+		> /dev/null 2>&1\
+	) && echo "snapshot ${ES_BACKUP_NAME} restored from elasticsearch repository" && touch elasticsearch-repository-restore
+
+elasticsearch-repository-check: elasticsearch-repository-config backup-dir
+	@if [ ! -f "${BACKUP_DIR}/${ES_BACKUP_NAME}.check" ]; then\
+		(\
+			docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch \
+				curl -s -XGET "localhost:9200/_snapshot/${APP_GROUP}/_all" \
+			| jq -r '.snapshots[].snapshot' | grep -q "${ES_BACKUP_NAME}" \
+		) > /dev/null 2>&1 \
+		&& echo "snapshot found for or ${ES_BACKUP_NAME} in elasticsearch repository" && mkdir -p touch "${BACKUP_DIR}/${ES_BACKUP_NAME}.check" \
+		|| (echo "no snapshot found for ${ES_BACKUP_NAME} in elasticsearch repository")\
+	fi
+
 elasticsearch-backup: elasticsearch-stop backup-dir
 	@echo taring ${ES_DATA} to ${BACKUP_DIR}/${ES_BACKUP_FILE}
 	@cd $$(dirname ${ES_DATA}) && sudo tar --create --file=${BACKUP_DIR}/${ES_BACKUP_FILE} --listed-incremental=${BACKUP_DIR}/${ES_BACKUP_FILE_SNAR} $$(basename ${ES_DATA})
@@ -227,9 +332,9 @@ endif
 
 elasticsearch-dev: elasticsearch
 
-elasticsearch: network vm_max
+elasticsearch-cluster: network vm_max
 	@echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes
-	@cat ${DC_FILE}-elasticsearch.yml | sed "s/%M/${ES_MEM}/g" > ${DC_FILE}-elasticsearch-huge.yml
+	@cat ${DC_FILE}-elasticsearch.yml > ${DC_FILE}-elasticsearch-huge.yml
 	@(if [ ! -d ${ES_DATA}/node1 ]; then sudo mkdir -p ${ES_DATA}/node1 ; sudo chmod g+rw ${ES_DATA}/node1/.; sudo chown 1000:1000 ${ES_DATA}/node1/.; fi)
 	@(i=$(ES_NODES); while [ $${i} -gt 1 ]; \
 		do \
@@ -241,15 +346,23 @@ elasticsearch: network vm_max
 	${DC} -f ${DC_FILE}-elasticsearch-huge.yml up -d
 	@timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0"  ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/_cat/indices > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ; echo ; exit $$ret
 
+elasticsearch-start: network vm_max
+	@echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes
+	@(if [ ! -d ${ES_DATA}/node1 ]; then sudo mkdir -p ${ES_DATA}/node1 ; sudo chmod g+rw ${ES_DATA}/node1/.; sudo chown 1000:1000 ${ES_DATA}/node1/.; fi)
+	${DC} -f ${DC_FILE}-elasticsearch.yml up -d
+
+elasticsearch: elasticsearch-start
+	@timeout=${TIMEOUT} ; ret=1 ; until [ "$$timeout" -le 0 -o "$$ret" -eq "0"  ] ; do (docker exec -i ${USE_TTY} ${DC_PREFIX}-elasticsearch curl -s --fail -XGET localhost:9200/ > /dev/null) ; ret=$$? ; if [ "$$ret" -ne "0" ] ; then echo -en "\rwaiting for elasticsearch API to start $$timeout" ; fi ; ((timeout--)); sleep 1 ; done ; echo ; exit $$ret
+
 elasticsearch2:
 	@echo docker-compose up matchID elasticsearch with ${ES_NODES} nodes
 	@cat ${DC_FILE}-elasticsearch.yml | head -8 > ${DC_FILE}-elasticsearch-huge-remote.yml
 	@(i=$$(( $(ES_NODES) * $(ES_SWARM_NODE_NUMBER) ));j=$$(( $(ES_NODES) * $(ES_SWARM_NODE_NUMBER) - $(ES_NODES))); while [ $${i} -gt $${j} ]; \
-	        do \
-	              if [ ! -d ${ES_DATA}/node$$i ]; then (echo ${ES_DATA}/node$$i && sudo mkdir -p ${ES_DATA}/node$$i && sudo chmod g+rw ${ES_DATA}/node$$i/. && sudo chown 1000:1000 ${ES_DATA}/node$$i/.); fi; \
-	              cat ${DC_FILE}-elasticsearch-node.yml | sed "s/%N/$$i/g;s/%MM/${ES_MEM}/g;s/%M/${ES_MEM}/g" | egrep -v 'depends_on|- elasticsearch' >> ${DC_FILE}-elasticsearch-huge-remote.yml; \
-	              i=`expr $$i - 1`; \
-	 	done;\
+		do \
+			if [ ! -d ${ES_DATA}/node$$i ]; then (echo ${ES_DATA}/node$$i && sudo mkdir -p ${ES_DATA}/node$$i && sudo chmod g+rw ${ES_DATA}/node$$i/. && sudo chown 1000:1000 ${ES_DATA}/node$$i/.); fi; \
+				cat ${DC_FILE}-elasticsearch-node.yml | sed "s/%N/$$i/g;s/%MM/${ES_MEM}/g;s/%M/${ES_MEM}/g" | egrep -v 'depends_on|- elasticsearch' >> ${DC_FILE}-elasticsearch-huge-remote.yml; \
+				i=`expr $$i - 1`; \
+		done;\
 	true)
 	${DC} -f ${DC_FILE}-elasticsearch-huge-remote.yml up -d
 

code/api.py

@@ -22,7 +22,6 @@
 import simplejson
 from collections.abc import Iterable
 from collections import OrderedDict
-from pandas.io.json import json_normalize
 from collections import deque
 
 
@@ -44,7 +43,7 @@
 from flask import Flask, current_app, jsonify, Response, abort, request, g, stream_with_context
 from flask.sessions import SecureCookieSessionInterface
 from flask_login import LoginManager, login_user, login_required, logout_user, current_user
-from flask_restplus import Resource, Api, reqparse
+from flask_restx import Resource, Api
 from werkzeug.utils import secure_filename
 from werkzeug.serving import run_simple
 from werkzeug.middleware.dispatcher import DispatcherMiddleware

code/parsers.py

@@ -1,5 +1,5 @@
 import werkzeug
-from flask_restplus import reqparse
+from flask_restx import reqparse
 from werkzeug.datastructures import FileStorage
 
 

code/recipes.py

@@ -27,7 +27,6 @@
 import simplejson
 from collections import Iterable
 from collections import OrderedDict
-from pandas.io.json import json_normalize
 from collections import deque
 
 
@@ -537,7 +536,7 @@ def iterator_from_files(self):
                         self.open(file),
                         sep=self.sep, usecols=self.select, chunksize=self.chunk,
                         encoding=self.encoding,dtype=object, header=self.header, names=self.names, skiprows=self.skiprows,
-                        prefix=self.prefix, iterator=True, index_col=False, keep_default_na=False
+                        iterator=True, index_col=False, keep_default_na=False
                     )
                 elif (self.type == "fwf"):
                     reader = pd.read_fwf(

docker-components/docker-compose-elasticsearch-node.yml

@@ -7,8 +7,9 @@
     environment:
       - cluster.name=matchid-cluster
       - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms%M -Xmx%M"
+      - ES_JAVA_OPTS
       - "discovery.zen.ping.unicast.hosts=elasticsearch"
+      - xpack.security.enabled=false
     ulimits:
       memlock:
         soft: -1

docker-components/docker-compose-elasticsearch-phonetic.yml

@@ -11,7 +11,7 @@ services:
     container_name: ${DC_PREFIX}-elasticsearch-phonetic
     environment:
       - cluster.name=docker-cluster
-      # - xpack.security.enabled=false
+      - xpack.security.enabled=false
       - "ES_JAVA_OPTS=-Xms3072m -Xmx3072m"
     ulimits:
       memlock:

docker-components/docker-compose-elasticsearch.yml

@@ -8,12 +8,27 @@ networks:
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
+    command:
+      - /bin/bash
+      - -c
+      - |
+        if [ ! -z "$http_proxy" ]; then
+          host=$${http_proxy##*//};
+          host=$${host%%:*};
+          port=$${http_proxy##*:};
+          (echo -e "cluster.name: \"docker-cluster\"\nnetwork.host: 0.0.0.0\ns3.client.default.proxy.host: $${host}\ns3.client.default.proxy.port: $${port}"  > /usr/share/elasticsearch/config/elasticsearch.yml);
+        fi;
+        exec /usr/local/bin/docker-entrypoint.sh elasticsearch
     container_name: ${DC_PREFIX}-elasticsearch
     environment:
       - cluster.name=matchid-cluster
       - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms%M -Xmx%M"
+      - ES_JAVA_OPTS
+      - http_proxy
+      - https_proxy
+      - no_proxy
       - discovery.type=single-node
+      - xpack.security.enabled=false
     ulimits:
       memlock:
         soft: -1

requirements.txt

@@ -4,7 +4,7 @@ elasticsearch
 enum34
 Flask
 flask-login
-flask_restplus
+flask_restx
 geopy
 fuzzywuzzy[speedup]
 jellyfish
@@ -21,5 +21,6 @@ smart-open==1.10.0
 sqlalchemy
 tslib
 typing
-Werkzeug==0.16.1
+Werkzeug==3.0.3
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability
+zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability