From 5e3318baaab85dcbca248bc24d397820f1a782fc Mon Sep 17 00:00:00 2001 From: mazano Date: Fri, 15 Sep 2023 17:29:44 +0200 Subject: [PATCH 1/5] fix #559 --- scripts/functions.sh | 13 +++++++++++-- scripts/start.sh | 11 +++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/functions.sh b/scripts/functions.sh index d3e9551d..aa14f895 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -414,8 +414,17 @@ function set_vars() { RANDOM_STRING=${RANDOMSTRING} fi - INSTANCE_STRING="${RANDOM_STRING}" - + # Search if cluster folder already exists + if [[ -d ${GEOSERVER_DATA_DIR}/cluster ]];then + # Search if cluster instance already exists + folder_path=$(find "${GEOSERVER_DATA_DIR}/cluster" -type d -name "instance_*" -print -quit) + if [ -n "$folder_path" ]; then + RANDOM_STRING=$(basename "$folder_path" | sed 's/^instance_//') + INSTANCE_STRING=${RANDOM_STRING} + fi + else + INSTANCE_STRING="${RANDOM_STRING}" + fi CLUSTER_CONFIG_DIR="${GEOSERVER_DATA_DIR}/cluster/instance_${RANDOM_STRING}" MONITOR_AUDIT_PATH="${GEOSERVER_DATA_DIR}/monitoring/monitor_${RANDOM_STRING}" diff --git a/scripts/start.sh b/scripts/start.sh index 963cbc3b..f8873850 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -175,10 +175,7 @@ if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then fi if [[ ! -f $CLUSTER_LOCKFILE ]]; then - if [[ -z "${EXISTING_DATA_DIR}" ]]; then - create_dir "${CLUSTER_CONFIG_DIR}" - fi - + create_dir "${CLUSTER_CONFIG_DIR}" if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]];then postgres_ssl_setup export SSL_PARAMETERS=${PARAMS} @@ -186,8 +183,10 @@ if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then broker_xml_config touch "${CLUSTER_LOCKFILE}" fi - # setup clustering if it's not already defined in an existing data directory - if [[ -z "${EXISTING_DATA_DIR}" ]]; then + # setup clustering if it's not already defined in an existing data directory + if find "$GEOSERVER_DATA_DIR" -type f \( -name "cluster.properties" -o -name "embedded-broker.properties" \); then + echo "Either cluster.properties or embedded-broker.properties is already defined in the data dir." + else cluster_config broker_config fi From d5d0cf7882d6b206b8b82a1e9ca3b929b6bae8e8 Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Sun, 17 Sep 2023 21:06:53 +0200 Subject: [PATCH 2/5] add clustering tests - a bit fragile cz of community plugin --- .github/workflows/build-latest.yaml | 1 + clustering/docker-compose.yml | 10 +- scenario_tests/clustering/docker-compose.yml | 88 +++++++++++++++++ scenario_tests/clustering/test.sh | 47 +++++++++ scenario_tests/clustering/tests/__init__.py | 0 scenario_tests/clustering/tests/init.sql | 20 ++++ scenario_tests/clustering/tests/test.sh | 17 ++++ .../tests/test_clustering_master.py | 98 +++++++++++++++++++ .../clustering/tests/test_clustering_node.py | 53 ++++++++++ scripts/entrypoint.sh | 20 ++-- scripts/functions.sh | 30 +++--- scripts/start.sh | 56 +++++------ 12 files changed, 381 insertions(+), 59 deletions(-) create mode 100644 scenario_tests/clustering/docker-compose.yml create mode 100755 scenario_tests/clustering/test.sh create mode 100644 scenario_tests/clustering/tests/__init__.py create mode 100644 scenario_tests/clustering/tests/init.sql create mode 100644 scenario_tests/clustering/tests/test.sh create mode 100644 scenario_tests/clustering/tests/test_clustering_master.py create mode 100644 scenario_tests/clustering/tests/test_clustering_node.py diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index c00a40fe..7435987c 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -48,6 +48,7 @@ jobs: - stores - context - disk-quota + - clustering steps: - uses: actions/checkout@v3 - name: Set up QEMU diff --git a/clustering/docker-compose.yml b/clustering/docker-compose.yml index 00d6eb68..3ee7ce1f 100644 --- a/clustering/docker-compose.yml +++ b/clustering/docker-compose.yml @@ -38,8 +38,8 @@ services: - POSTGRES_USER=docker - POSTGRES_PASS=docker - SSL_MODE=allow - - GEOSERVER_ADMIN_PASSWORD=admin - - GEOSERVER_ADMIN_USER=myawesomegeoserver + - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver + - GEOSERVER_ADMIN_USER=admin restart: on-failure depends_on: db: @@ -69,12 +69,12 @@ services: - POSTGRES_USER=docker - POSTGRES_PASS=docker - SSL_MODE=allow - - GEOSERVER_ADMIN_PASSWORD=admin - - GEOSERVER_ADMIN_USER=myawesomegeoserver + - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver + - GEOSERVER_ADMIN_USER=admin restart: on-failure depends_on: master: - condition: service_healthy + condition: service_started db: condition: service_healthy healthcheck: diff --git a/scenario_tests/clustering/docker-compose.yml b/scenario_tests/clustering/docker-compose.yml new file mode 100644 index 00000000..43789f96 --- /dev/null +++ b/scenario_tests/clustering/docker-compose.yml @@ -0,0 +1,88 @@ +version: '3.9' + +volumes: + geoserver-cluster-data: + geo-db-data: + +services: + db: + image: kartoza/postgis:15-3.4 + volumes: + - geo-db-data:/var/lib/postgresql + - ./tests/init.sql:/docker-entrypoint-initdb.d/init.sql + environment: + - POSTGRES_DB=gis + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - ALLOW_IP_RANGE=0.0.0.0/0 + - FORCE_SSL=FALSE + restart: on-failure + healthcheck: + test: "PGPASSWORD=docker pg_isready -h 127.0.0.1 -U docker -d gis" + master: + image: 'kartoza/geoserver:${TAG:-manual-build}' + volumes: + - geoserver-cluster-data:/opt/geoserver/data_dir + - ./tests:/tests + environment: + - BROKER_URL=tcp://0.0.0.0:61661 + - READONLY=disabled + - CLUSTER_DURABILITY=false + - CLUSTERING=True + - TOGGLE_MASTER=true + - TOGGLE_SLAVE=true + - DB_BACKEND=POSTGRES + - HOST=db + - POSTGRES_PORT=5432 + - POSTGRES_DB=gis + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - SSL_MODE=allow + - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver + - GEOSERVER_ADMIN_USER=admin + - TEST_CLASS=test_clustering_master.GeoServerClusteringMaster + restart: on-failure + depends_on: + db: + condition: service_healthy + healthcheck: + test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u admin:'myawesomegeoserver' http://localhost:8080/geoserver/rest/about/version.xml" + interval: 1m30s + timeout: 10s + retries: 3 + node: + image: 'kartoza/geoserver:${TAG:-manual-build}' + volumes: + - geoserver-cluster-data:/opt/geoserver/data_dir + - ./tests:/tests + environment: + - BROKER_URL=tcp://master:61661 + - READONLY=disabled + - CLUSTER_DURABILITY=false + - CLUSTERING=True + - TOGGLE_MASTER=true + - TOGGLE_SLAVE=true + - EMBEDDED_BROKER=disabled + - HOST=db + - POSTGRES_PORT=5432 + - POSTGRES_DB=gis + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - SSL_MODE=allow + - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver + - GEOSERVER_ADMIN_USER=admin + - TEST_CLASS=test_clustering_node.GeoServerClusteringNode + restart: on-failure + depends_on: + master: + condition: service_started + db: + condition: service_healthy + healthcheck: + test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u admin:'myawesomegeoserver' http://localhost:8080/geoserver/rest/about/version.xml" + interval: 1m30s + timeout: 10s + retries: 3 + + + diff --git a/scenario_tests/clustering/test.sh b/scenario_tests/clustering/test.sh new file mode 100755 index 00000000..6cc5f5d3 --- /dev/null +++ b/scenario_tests/clustering/test.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +if [[ $(dpkg -l | grep "docker-compose") > /dev/null ]];then + VERSION='docker-compose' + else + VERSION='docker compose' +fi + +${VERSION} -f docker-compose.yml up -d + +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + ${VERSION} -f docker-compose.yml logs -f & +fi + +sleep 60 + +# Test Master +services=("master") + +for service in "${services[@]}"; do + + # Execute tests + sleep 60 + echo "Execute test for $service" + ${VERSION} -f docker-compose.yml exec "${service}" /bin/bash /tests/test.sh + +done + +# Test Node +services=("node") + +for service in "${services[@]}"; do + + # Execute tests + sleep 60 + echo "Execute test for $service" + ${VERSION} -f docker-compose.yml exec "${service}" /bin/bash /tests/test.sh + +done + +${VERSION} -f docker-compose.yml down -v diff --git a/scenario_tests/clustering/tests/__init__.py b/scenario_tests/clustering/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scenario_tests/clustering/tests/init.sql b/scenario_tests/clustering/tests/init.sql new file mode 100644 index 00000000..072e2927 --- /dev/null +++ b/scenario_tests/clustering/tests/init.sql @@ -0,0 +1,20 @@ + CREATE TABLE IF NOT EXISTS states ( + id integer not null + constraint pkey primary key, + geom geometry(Point, 4326), + name varchar(30), + alias varchar(30), + description varchar(255) + ); + +INSERT INTO states (id, geom, name, alias, description) + VALUES + ( + 1, + st_setsrid(st_point(107.6097, 6.9120), 4326), + 'Bandung', + 'Paris van Java', + 'Asia-Africa conference was held here' + ) ON CONFLICT DO NOTHING; + + diff --git a/scenario_tests/clustering/tests/test.sh b/scenario_tests/clustering/tests/test.sh new file mode 100644 index 00000000..7337b414 --- /dev/null +++ b/scenario_tests/clustering/tests/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +cat << EOF +Settings used: + +GEOSERVER_ADMIN_PASSWORD: ${GEOSERVER_ADMIN_PASSWORD} +GEOSERVER_ADMIN_USER: ${GEOSERVER_ADMIN_USER} +EOF + +python3 -m unittest -v ${TEST_CLASS} diff --git a/scenario_tests/clustering/tests/test_clustering_master.py b/scenario_tests/clustering/tests/test_clustering_master.py new file mode 100644 index 00000000..9da601f2 --- /dev/null +++ b/scenario_tests/clustering/tests/test_clustering_master.py @@ -0,0 +1,98 @@ +import unittest +from os import environ + +from geo.Geoserver import Geoserver +from requests import get, post, exceptions +from requests.auth import HTTPBasicAuth + + +class GeoServerClusteringMaster(unittest.TestCase): + + def setUp(self): + # Define the GeoServer URL + self.gs_url = 'http://localhost:8080/geoserver' + # Define the PostGIS JNDI store name + self.store_name = 'gis' + self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin') + self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver') + self.geo_workspace_name = 'demo' + + def test_publish_store(self): + geo = Geoserver(self.gs_url, username='%s' % self.geo_username, password='%s' % self.geo_password) + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + # create workspace + try: + rest_url = '%s/rest/workspaces/%s.json' % (self.gs_url, self.geo_workspace_name) + response = get(rest_url, auth=auth) + response.raise_for_status() + except exceptions.HTTPError: + geo.create_workspace(workspace='%s' % self.geo_workspace_name) + geo.set_default_workspace('%s' % self.geo_workspace_name) + + # Create the XML payload for the JNDI store configuration + xml = """ + + {name} + PostGIS + true + + db + 5432 + gis + docker + docker + public + true + 1000 + true + false + true + 1 + false + FAST + postgis + true + + false + + """.format(name=self.store_name) + + # Publish the store + response = post(self.gs_url + '/rest/workspaces/%s/datastores' % self.geo_workspace_name, auth=auth, + headers={'Content-type': 'text/xml'}, + data=xml) + + # Check that the response has a status code of 201 (Created) + self.assertEqual(response.status_code, 201) + + # Check that the store exists + data_source_url = '%s/rest/workspaces/%s/datastores/%s.json' % ( + self.gs_url, self.geo_workspace_name, self.store_name) + response = get(data_source_url, auth=auth) + + # Check that the response has a status code of 200 (OK) + self.assertEqual(response.status_code, 200) + + # Publish layer into GeoServer + try: + rest_url = '%s/rest/workspaces/%s/datastores/%s/featuretypes/states.json' % ( + self.gs_url, self.geo_workspace_name, self.store_name) + publish_response = get(rest_url, auth=auth) + publish_response.raise_for_status() + except exceptions.HTTPError: + geo.publish_featurestore(workspace='%s' + % self.geo_workspace_name, store_name='%s' % self.store_name, + pg_table='states') + self.assertEqual(response.status_code, 200) + + # Check that the layer exists + layer_url = '%s/rest/workspaces/%s/layers/states.json' % ( + self.gs_url, self.geo_workspace_name) + response = get(layer_url, auth=auth) + + # Check that the response has a status code of 200 (OK) + self.assertEqual(response.status_code, 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/scenario_tests/clustering/tests/test_clustering_node.py b/scenario_tests/clustering/tests/test_clustering_node.py new file mode 100644 index 00000000..3f9ef51b --- /dev/null +++ b/scenario_tests/clustering/tests/test_clustering_node.py @@ -0,0 +1,53 @@ +import unittest +from os import environ +from requests import get, exceptions +from requests.auth import HTTPBasicAuth + + +class GeoServerClusteringNode(unittest.TestCase): + + def setUp(self): + # Define the GeoServer URL + self.gs_url = 'http://localhost:8080/geoserver' + # Define the PostGIS store name + self.store_name = 'gis' + self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin') + self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver') + self.geo_workspace_name = 'demo' + + def check_workspace_exists(self, auth): + rest_url = '%s/rest/workspaces/%s.json' % (self.gs_url, self.geo_workspace_name) + response = get(rest_url, auth=auth) + return response.status_code == 200 + + def check_data_store_exists(self, auth): + data_source_url = '%s/rest/workspaces/%s/datastores/%s.json' % ( + self.gs_url, self.geo_workspace_name, self.store_name) + response = get(data_source_url, auth=auth) + return response.status_code == 200 + + def check_layer_exists(self, auth, layer_name): + layer_url = '%s/rest/workspaces/%s/layers/%s.json' % ( + self.gs_url, self.geo_workspace_name, layer_name) + response = get(layer_url, auth=auth) + return response.status_code == 200 + + def test_workspace_exists(self): + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + workspace_exists = self.check_workspace_exists(auth) + self.assertTrue(workspace_exists, "Workspace does not exist") + + def test_data_store_exists(self): + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + data_store_exists = self.check_data_store_exists(auth) + self.assertTrue(data_store_exists, "Data store does not exist") + + def test_layer_exists(self): + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + layer_name = 'states' + layer_exists = self.check_layer_exists(auth, layer_name) + self.assertTrue(layer_exists, "Layer does not exist") + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 7190be70..4c591d49 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -11,8 +11,8 @@ USER_NAME=${USER:-geoserveruser} GEO_GROUP_NAME=${GROUP_NAME:-geoserverusers} # Add group -if [ ! $(getent group "${GEO_GROUP_NAME}") ]; then - groupadd -r "${GEO_GROUP_NAME}" -g ${GROUP_ID} +if [ ! "$(getent group "${GEO_GROUP_NAME}")" ]; then + groupadd -r "${GEO_GROUP_NAME}" -g "${GROUP_ID}" fi # Add user to system @@ -45,7 +45,7 @@ fi # Credits https://github.com/kartoza/docker-geoserver/pull/371 set_vars export READONLY CLUSTER_DURABILITY BROKER_URL EMBEDDED_BROKER TOGGLE_MASTER TOGGLE_SLAVE BROKER_URL -export CLUSTER_CONFIG_DIR MONITOR_AUDIT_PATH CLUSTER_LOCKFILE INSTANCE_STRING +export CLUSTER_CONFIG_DIR MONITOR_AUDIT_PATH INSTANCE_STRING CLUSTER_CONNECTION_RETRY_COUNT CLUSTER_CONNECTION_MAX_WAIT /bin/bash /scripts/start.sh @@ -97,18 +97,18 @@ export JAVA_OPTS="${JAVA_OPTS} ${GEOSERVER_OPTS}" # Chown again - seems to fix issue with resolving all created directories if [[ ${RUN_AS_ROOT} =~ [Ff][Aa][Ll][Ss][Ee] ]];then - dir_ownership=(${CATALINA_HOME} /home/"${USER_NAME}"/ "${COMMUNITY_PLUGINS_DIR}" + dir_ownership=("${CATALINA_HOME}" /home/"${USER_NAME}"/ "${COMMUNITY_PLUGINS_DIR}" "${STABLE_PLUGINS_DIR}" "${GEOSERVER_HOME}" /usr/share/fonts/ /tomcat_apps.zip /tmp/ "${FOOTPRINTS_DATA_DIR}" "${CERT_DIR}" "${FONTS_DIR}" /scripts/ "${EXTRA_CONFIG_DIR}") for directory in "${dir_ownership[@]}"; do - if [[ $(stat -c '%U' ${directory}) != "${USER_NAME}" ]] && [[ $(stat -c '%G' ${directory}) != "${GEO_GROUP_NAME}" ]];then - chown -R "${USER_NAME}":"${GEO_GROUP_NAME}" ${directory} + if [[ $(stat -c '%U' "${directory}") != "${USER_NAME}" ]] && [[ $(stat -c '%G' "${directory}") != "${GEO_GROUP_NAME}" ]];then + chown -R "${USER_NAME}":"${GEO_GROUP_NAME}" "${directory}" fi done fi -chmod o+rw "${CERT_DIR}";gwc_file_perms ;chmod 400 ${CATALINA_HOME}/conf/* +chmod o+rw "${CERT_DIR}";gwc_file_perms ;chmod 400 "${CATALINA_HOME}"/conf/* if [[ ${SAMPLE_DATA} =~ [Tt][Rr][Uu][Ee] ]]; then chown -R "${USER_NAME}":"${GEO_GROUP_NAME}" "${GEOSERVER_DATA_DIR}" @@ -116,13 +116,13 @@ fi if [[ ${RUN_AS_ROOT} =~ [Ff][Aa][Ll][Ss][Ee] ]];then if [[ -f ${GEOSERVER_HOME}/start.jar ]]; then - exec gosu ${USER_NAME} ${GEOSERVER_HOME}/bin/startup.sh + exec gosu "${USER_NAME}" "${GEOSERVER_HOME}"/bin/startup.sh else - exec gosu ${USER_NAME} /usr/local/tomcat/bin/catalina.sh run + exec gosu "${USER_NAME}" /usr/local/tomcat/bin/catalina.sh run fi else if [[ -f ${GEOSERVER_HOME}/start.jar ]]; then - exec ${GEOSERVER_HOME}/bin/startup.sh + exec "${GEOSERVER_HOME}"/bin/startup.sh else exec /usr/local/tomcat/bin/catalina.sh run fi diff --git a/scripts/functions.sh b/scripts/functions.sh index 91a6b8b4..6034c22f 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -12,7 +12,7 @@ function validate_url(){ if [ -n "$2" ]; then EXTRA_PARAMS=$2 fi - if [[ `wget -S --spider $1 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then + if [[ $(wget -S --spider $1 2>&1 | grep 'HTTP/1.1 200 OK') ]]; then ${request} "${1}" "${2}" else echo -e "URL : \e[1;31m $1 does not exists \033[0m" @@ -225,6 +225,10 @@ fi # Helper function to setup cluster config for the clustering plugin # https://docs.geoserver.org/stable/en/user/community/jms-cluster/index.html function cluster_config() { + # Remove default config + if [ -f "${CLUSTER_CONFIG_DIR}"/cluster.properties ];then + rm "${CLUSTER_CONFIG_DIR}"/cluster.properties + fi if [[ ! -f ${CLUSTER_CONFIG_DIR}/cluster.properties ]]; then # If it doesn't exists, copy from /settings directory if exists if [[ -f ${EXTRA_CONFIG_DIR}/cluster.properties ]]; then @@ -240,6 +244,11 @@ function cluster_config() { # https://docs.geoserver.org/stable/en/user/community/jms-cluster/index.html function broker_config() { + # Delete default config + if [ -f "${CLUSTER_CONFIG_DIR}"/embedded-broker.properties ];then + rm "${CLUSTER_CONFIG_DIR}"/embedded-broker.properties + fi + if [[ ! -f ${CLUSTER_CONFIG_DIR}/embedded-broker.properties ]]; then # If it doesn't exists, copy from /settings directory if exists if [[ -f ${EXTRA_CONFIG_DIR}/embedded-broker.properties ]]; then @@ -252,6 +261,10 @@ function broker_config() { } function broker_xml_config() { + # Delete default config + if [ -f "${CLUSTER_CONFIG_DIR}"/broker.xml ];then + rm "${CLUSTER_CONFIG_DIR}"/broker.xml + fi if [[ ! -f ${CLUSTER_CONFIG_DIR}/broker.xml ]]; then # If it doesn't exists, copy from /settings directory if exists if [[ -f ${EXTRA_CONFIG_DIR}/broker.xml ]]; then @@ -425,26 +438,13 @@ function set_vars() { RANDOM_STRING=${RANDOMSTRING} fi - # Search if cluster folder already exists - if [[ -d ${GEOSERVER_DATA_DIR}/cluster ]];then - # Search if cluster instance already exists - folder_path=$(find "${GEOSERVER_DATA_DIR}/cluster" -type d -name "instance_*" -print -quit) - if [ -n "$folder_path" ]; then - RANDOM_STRING=$(basename "$folder_path" | sed 's/^instance_//') - INSTANCE_STRING=${RANDOM_STRING} - fi - else - INSTANCE_STRING="${RANDOM_STRING}" - fi - + INSTANCE_STRING="${RANDOM_STRING}" CLUSTER_CONFIG_DIR="${GEOSERVER_DATA_DIR}/cluster/instance_${RANDOM_STRING}" MONITOR_AUDIT_PATH="${GEOSERVER_DATA_DIR}/monitoring/monitor_${RANDOM_STRING}" - CLUSTER_LOCKFILE="${CLUSTER_CONFIG_DIR}/.cluster.lock" } - function postgres_ssl_setup() { if [[ ${SSL_MODE} == 'verify-ca' || ${SSL_MODE} == 'verify-full' ]]; then if [[ -z ${SSL_CERT_FILE} || -z ${SSL_KEY_FILE} || -z ${SSL_CA_FILE} ]]; then diff --git a/scripts/start.sh b/scripts/start.sh index f8873850..2dbfe392 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -9,7 +9,7 @@ web_cors # Useful for development - We need a clean state of data directory if [[ "${RECREATE_DATADIR}" =~ [Tt][Rr][Uu][Ee] ]]; then - rm -rf "${GEOSERVER_DATA_DIR}"/* + rm -rf "${GEOSERVER_DATA_DIR:?}/"* fi # install Font files in resources/fonts if they exists @@ -29,9 +29,6 @@ create_dir "${GEOWEBCACHE_CACHE_DIR}" setup_custom_crs setup_custom_override_crs -create_dir "${GEOSERVER_DATA_DIR}"/logs -export GEOSERVER_LOG_LEVEL -geoserver_logging # Activate sample data if [[ ${SAMPLE_DATA} =~ [Tt][Rr][Uu][Ee] ]]; then @@ -60,9 +57,10 @@ if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]]; then echo -e "[Entrypoint] Checking PostgreSQL connection to see if diskquota tables are loaded: \033[0m" if [[ ${POSTGRES_SCHEMA} != 'public' ]]; then - export PGPASSWORD="${POSTGRES_PASS}" - postgres_ready_status ${HOST} ${POSTGRES_PORT} ${POSTGRES_USER} $POSTGRES_DB - create_gwc_tile_tables ${HOST} ${POSTGRES_PORT} ${POSTGRES_USER} $POSTGRES_DB $POSTGRES_SCHEMA + PGPASSWORD="${POSTGRES_PASS}" + export PGPASSWORD + postgres_ready_status "${HOST}" "${POSTGRES_PORT}" "${POSTGRES_USER}" "$POSTGRES_DB" + create_gwc_tile_tables "${HOST}" "${POSTGRES_PORT}" "${POSTGRES_USER}" "$POSTGRES_DB" "$POSTGRES_SCHEMA" fi else export DISK_QUOTA_BACKEND=H2 @@ -151,7 +149,7 @@ fi # Setup clustering set_vars export READONLY CLUSTER_DURABILITY BROKER_URL EMBEDDED_BROKER TOGGLE_MASTER TOGGLE_SLAVE BROKER_URL -export CLUSTER_CONFIG_DIR MONITOR_AUDIT_PATH CLUSTER_LOCKFILE INSTANCE_STRING +export CLUSTER_CONFIG_DIR MONITOR_AUDIT_PATH INSTANCE_STRING create_dir "${MONITOR_AUDIT_PATH}" if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then @@ -174,30 +172,30 @@ if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then fi - if [[ ! -f $CLUSTER_LOCKFILE ]]; then + + if [[ ! -d "${CLUSTER_CONFIG_DIR}" ]];then create_dir "${CLUSTER_CONFIG_DIR}" - if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]];then - postgres_ssl_setup - export SSL_PARAMETERS=${PARAMS} - fi - broker_xml_config - touch "${CLUSTER_LOCKFILE}" - fi - # setup clustering if it's not already defined in an existing data directory - if find "$GEOSERVER_DATA_DIR" -type f \( -name "cluster.properties" -o -name "embedded-broker.properties" \); then - echo "Either cluster.properties or embedded-broker.properties is already defined in the data dir." - else - cluster_config - broker_config fi + if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]];then + postgres_ssl_setup + export SSL_PARAMETERS=${PARAMS} + fi + # Setup configs + broker_xml_config + cluster_config + broker_config + fi export REQUEST_TIMEOUT PARALLEL_REQUEST GETMAP REQUEST_EXCEL SINGLE_USER GWC_REQUEST WPS_REQUEST # Setup control flow properties setup_control_flow +create_dir "${GEOSERVER_DATA_DIR}"/logs +export GEOSERVER_LOG_LEVEL +geoserver_logging if [[ ${POSTGRES_JNDI} =~ [Tt][Rr][Uu][Ee] ]];then postgres_ssl_setup @@ -208,7 +206,7 @@ if [[ ${POSTGRES_JNDI} =~ [Tt][Rr][Uu][Ee] ]];then fi POSTGRES_JAR_COUNT=$(ls -1 ${CATALINA_HOME}/webapps/${GEOSERVER_CONTEXT_ROOT}/WEB-INF/lib/postgresql-* 2>/dev/null | wc -l) if [ "$POSTGRES_JAR_COUNT" != 0 ]; then - rm "${CATALINA_HOME}"/webapps/${GEOSERVER_CONTEXT_ROOT}/WEB-INF/lib/postgresql-* + rm "${CATALINA_HOME}"/webapps/"${GEOSERVER_CONTEXT_ROOT}"/WEB-INF/lib/postgresql-* fi cp "${CATALINA_HOME}"/postgres_config/postgresql-* "${CATALINA_HOME}"/lib/ if [[ -f ${EXTRA_CONFIG_DIR}/context.xml ]]; then @@ -219,7 +217,7 @@ if [[ ${POSTGRES_JNDI} =~ [Tt][Rr][Uu][Ee] ]];then fi else - cp "${CATALINA_HOME}"/postgres_config/postgresql-* "${CATALINA_HOME}"/webapps/${GEOSERVER_CONTEXT_ROOT}/WEB-INF/lib/ + cp "${CATALINA_HOME}"/postgres_config/postgresql-* "${CATALINA_HOME}"/webapps/"${GEOSERVER_CONTEXT_ROOT}"/WEB-INF/lib/ fi @@ -229,7 +227,7 @@ if [[ "${TOMCAT_EXTRAS}" =~ [Tt][Rr][Uu][Ee] ]]; then rm -r /tmp/tomcat_apps if [[ ${POSTGRES_JNDI} =~ [Ff][Aa][Ll][Ss][Ee] ]]; then if [[ -f ${EXTRA_CONFIG_DIR}/context.xml ]]; then - envsubst < ${EXTRA_CONFIG_DIR}/context.xml > "${CATALINA_HOME}"/webapps/manager/META-INF/context.xml + envsubst < "${EXTRA_CONFIG_DIR}"/context.xml > "${CATALINA_HOME}"/webapps/manager/META-INF/context.xml else cp /build_data/context.xml "${CATALINA_HOME}"/webapps/manager/META-INF/ sed -i -e '19,36d' "${CATALINA_HOME}"/webapps/manager/META-INF/context.xml @@ -238,7 +236,7 @@ if [[ "${TOMCAT_EXTRAS}" =~ [Tt][Rr][Uu][Ee] ]]; then if [[ -z ${TOMCAT_PASSWORD} ]]; then generate_random_string 18 export TOMCAT_PASSWORD=${RAND} - echo $TOMCAT_PASSWORD >${GEOSERVER_DATA_DIR}/security/tomcat_pass.txt + echo "${TOMCAT_PASSWORD}" >"${GEOSERVER_DATA_DIR}"/tomcat_pass.txt if [[ ${SHOW_PASSWORD} =~ [Tt][Rr][Uu][Ee] ]];then echo -e "[Entrypoint] GENERATED tomcat PASSWORD: \e[1;31m $TOMCAT_PASSWORD \033[0m" fi @@ -267,7 +265,7 @@ if [[ ${SSL} =~ [Tt][Rr][Uu][Ee] ]]; then # convert LetsEncrypt certificates # https://community.letsencrypt.org/t/cry-for-help-windows-tomcat-ssl-lets-encrypt/22902/4 - # remove existing keystores + # remove existing key-stores rm -f "$P12_FILE" rm -f "$JKS_FILE" @@ -442,13 +440,13 @@ else eval "$transform" # Add x-forwarded headers if [[ "${ACTIVATE_PROXY_HEADERS}" =~ [Tt][Rr][Uu][Ee] ]]; then - sed -i.bak -r '/\<\Host\>/ i\ \t' ${CATALINA_HOME}/conf/server.xml + sed -i.bak -r '/\<\Host\>/ i\ \t' "${CATALINA_HOME}"/conf/server.xml fi fi # Cleanup temp file -delete_file ${CATALINA_HOME}/conf/ssl-tomcat_no_https.xsl +delete_file "${CATALINA_HOME}"/conf/ssl-tomcat_no_https.xsl if [[ -z "${EXISTING_DATA_DIR}" ]]; then From a4c31bf8db3d27fc39dc0d354f4e7b31c23ac4eb Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Mon, 18 Sep 2023 21:06:14 +0200 Subject: [PATCH 3/5] fixes #514 --- scripts/start.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/start.sh b/scripts/start.sh index 2dbfe392..bbcc7f9d 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -186,7 +186,10 @@ if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then broker_xml_config cluster_config broker_config - + # Download Clustering module, temporary fixes https://github.com/kartoza/docker-geoserver/issues/514 + ${request} https://download.jar-download.com/cache_jars/org.jdom/jdom2/2.0.6.1/jar_files.zip + unzip jar_files.zip -d "${CATALINA_HOME}"/webapps/"${GEOSERVER_CONTEXT_ROOT}"/WEB-INF/lib/ + rm jar_files.zip fi export REQUEST_TIMEOUT PARALLEL_REQUEST GETMAP REQUEST_EXCEL SINGLE_USER GWC_REQUEST WPS_REQUEST From 7684704d8b23ca9fc22a9d741099cc02cd293f0d Mon Sep 17 00:00:00 2001 From: mazano Date: Tue, 19 Sep 2023 17:14:59 +0200 Subject: [PATCH 4/5] Add option to check for an existing data dir in cluster init --- scripts/entrypoint.sh | 14 +++++++------ scripts/start.sh | 49 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 4c591d49..e5a98f60 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -20,15 +20,17 @@ if ! id -u "${USER_NAME}" >/dev/null 2>&1; then useradd -l -m -d /home/"${USER_NAME}"/ -u "${USER_ID}" --gid "${GROUP_ID}" -s /bin/bash -G "${GEO_GROUP_NAME}" "${USER_NAME}" fi -# Create directories -mkdir -p "${GEOSERVER_DATA_DIR}" "${CERT_DIR}" "${FOOTPRINTS_DATA_DIR}" "${FONTS_DIR}" "${GEOWEBCACHE_CACHE_DIR}" \ -"${GEOSERVER_HOME}" "${EXTRA_CONFIG_DIR}" - - - +# Import env and functions source /scripts/functions.sh source /scripts/env-data.sh +# Create directories +dir_creation=("${GEOSERVER_DATA_DIR}" "${CERT_DIR}" "${FOOTPRINTS_DATA_DIR}" "${FONTS_DIR}" "${GEOWEBCACHE_CACHE_DIR}" +"${GEOSERVER_HOME}" "${EXTRA_CONFIG_DIR}") +for directory in "${dir_creation[@]}"; do + create_dir "${directory}" +done + # Rename to match wanted context-root and so that we can unzip plugins to # existing directory. if [ x"${GEOSERVER_CONTEXT_ROOT}" != xgeoserver ]; then diff --git a/scripts/start.sh b/scripts/start.sh index bbcc7f9d..7cb8d4e6 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -172,20 +172,49 @@ if [[ ${CLUSTERING} =~ [Tt][Rr][Uu][Ee] ]]; then fi + if [[ -z "${EXISTING_DATA_DIR}" ]];then + if [[ ! -d "${CLUSTER_CONFIG_DIR}" ]];then + create_dir "${CLUSTER_CONFIG_DIR}" + fi + if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]];then + postgres_ssl_setup + export SSL_PARAMETERS=${PARAMS} + fi + # Setup configs + broker_xml_config + cluster_config + broker_config + else + if [[ -z "${CLUSTER_CONFIG_DIR}" ]];then + echo -e "\e[32m -------------------------------------------------------------------------------- \033[0m" + echo -e "[Entrypoint] You are using an existing data directory but you haven't set : \e[1;31m $CLUSTER_CONFIG_DIR \033[0m" + exit 1 + else + # Variable to count files if found + count=0 - if [[ ! -d "${CLUSTER_CONFIG_DIR}" ]];then - create_dir "${CLUSTER_CONFIG_DIR}" - fi + # Check if cluster.properties exists and increment the count if found + if find "${CLUSTER_CONFIG_DIR}" -type f -name "cluster.properties" -exec test -f {} \; ; then + count=$((count + 1)) + fi + # Check if embedded-broker.properties exists and increment the count if found + if find "${CLUSTER_CONFIG_DIR}" -type f -name "embedded-broker.properties" -exec test -f {} \; ; then + count=$((count + 1)) + fi + + # Check if broker.xml exists and increment the count if found + if find "${CLUSTER_CONFIG_DIR}" -type f -name "broker.xml" -exec test -f {} \; ; then + count=$((count + 1)) + fi - if [[ ${DB_BACKEND} =~ [Pp][Oo][Ss][Tt][Gg][Rr][Ee][Ss] ]];then - postgres_ssl_setup - export SSL_PARAMETERS=${PARAMS} + # Check if all three files were found + if [ $count -ne 3 ]; then + echo "cluster.properties,embedded-broker.properties and broker.xml were not found in ${CLUSTER_CONFIG_DIR} exiting." + exit 1 + fi + fi fi - # Setup configs - broker_xml_config - cluster_config - broker_config # Download Clustering module, temporary fixes https://github.com/kartoza/docker-geoserver/issues/514 ${request} https://download.jar-download.com/cache_jars/org.jdom/jdom2/2.0.6.1/jar_files.zip unzip jar_files.zip -d "${CATALINA_HOME}"/webapps/"${GEOSERVER_CONTEXT_ROOT}"/WEB-INF/lib/ From 3d5f2ea2be95cd0582b5115b44a4967cb7fa68cd Mon Sep 17 00:00:00 2001 From: mazano Date: Tue, 19 Sep 2023 17:18:50 +0200 Subject: [PATCH 5/5] add info to readme --- clustering/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clustering/README.md b/clustering/README.md index 9adae17d..1ed7975a 100644 --- a/clustering/README.md +++ b/clustering/README.md @@ -17,4 +17,7 @@ This value will be different for (Master-Node) * `TOGGLE_SLAVE=true` - Differentiates if the instance will be a Node * `EMBEDDED_BROKER=disabled` - Should be disabled for the Node * `CLUSTER_CONNECTION_RETRY_COUNT=10` - How many times try to connect to broker -* `CLUSTER_CONNECTION_MAX_WAIT=500` - Wait time between connection to broker retry (in milliseconds) \ No newline at end of file +* `CLUSTER_CONNECTION_MAX_WAIT=500` - Wait time between connection to broker retry (in milliseconds) +* `EXISTING_DATA_DIR` - If you are using an existing data directory, you need to set `CLUSTER_CONFIG_DIR` +otherwise the container is will hang and not start. Additionally, it will check if all the files +needed for clustering exists, otherwise it will fail. \ No newline at end of file