fix #559

.github/workflows/build-latest.yaml

@@ -48,6 +48,7 @@ jobs:
           - stores
           - context
           - disk-quota
+          - clustering
     steps:
       - uses: actions/checkout@v3
       - name: Set up QEMU

clustering/README.md

@@ -17,4 +17,7 @@ This value will be different for (Master-Node)
 * `TOGGLE_SLAVE=true` - Differentiates if the instance will be a Node
 * `EMBEDDED_BROKER=disabled` - Should be disabled for the Node
 * `CLUSTER_CONNECTION_RETRY_COUNT=10` - How many times try to connect to broker
-* `CLUSTER_CONNECTION_MAX_WAIT=500` - Wait time between connection to broker retry (in milliseconds)
+* `CLUSTER_CONNECTION_MAX_WAIT=500` - Wait time between connection to broker retry (in milliseconds)
+* `EXISTING_DATA_DIR` - If you are using an existing data directory, you need to set `CLUSTER_CONFIG_DIR`
+otherwise the container is will hang and not start. Additionally, it will check if all the files
+needed for clustering exists, otherwise it will fail.

clustering/docker-compose.yml

@@ -38,8 +38,8 @@ services:
         - POSTGRES_USER=docker
         - POSTGRES_PASS=docker
         - SSL_MODE=allow
-        - GEOSERVER_ADMIN_PASSWORD=admin
-        - GEOSERVER_ADMIN_USER=myawesomegeoserver
+        - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver
+        - GEOSERVER_ADMIN_USER=admin
       restart: on-failure
       depends_on:
         db:
@@ -69,12 +69,12 @@ services:
         - POSTGRES_USER=docker
         - POSTGRES_PASS=docker
         - SSL_MODE=allow
-        - GEOSERVER_ADMIN_PASSWORD=admin
-        - GEOSERVER_ADMIN_USER=myawesomegeoserver
+        - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver
+        - GEOSERVER_ADMIN_USER=admin
       restart: on-failure
       depends_on:
         master:
-          condition: service_healthy
+          condition: service_started
         db:
           condition: service_healthy
       healthcheck:

scenario_tests/clustering/docker-compose.yml

@@ -0,0 +1,88 @@
+version: '3.9'
+
+volumes:
+   geoserver-cluster-data:
+   geo-db-data:
+
+services:
+   db:
+      image: kartoza/postgis:15-3.4
+      volumes:
+        - geo-db-data:/var/lib/postgresql
+        - ./tests/init.sql:/docker-entrypoint-initdb.d/init.sql
+      environment:
+        - POSTGRES_DB=gis
+        - POSTGRES_USER=docker
+        - POSTGRES_PASS=docker
+        - ALLOW_IP_RANGE=0.0.0.0/0
+        - FORCE_SSL=FALSE
+      restart: on-failure
+      healthcheck:
+        test: "PGPASSWORD=docker pg_isready -h 127.0.0.1 -U docker -d gis"
+   master:
+      image: 'kartoza/geoserver:${TAG:-manual-build}'
+      volumes:
+         - geoserver-cluster-data:/opt/geoserver/data_dir
+         - ./tests:/tests
+      environment:
+        - BROKER_URL=tcp://0.0.0.0:61661
+        - READONLY=disabled
+        - CLUSTER_DURABILITY=false
+        - CLUSTERING=True
+        - TOGGLE_MASTER=true
+        - TOGGLE_SLAVE=true
+        - DB_BACKEND=POSTGRES
+        - HOST=db
+        - POSTGRES_PORT=5432
+        - POSTGRES_DB=gis
+        - POSTGRES_USER=docker
+        - POSTGRES_PASS=docker
+        - SSL_MODE=allow
+        - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver
+        - GEOSERVER_ADMIN_USER=admin
+        - TEST_CLASS=test_clustering_master.GeoServerClusteringMaster
+      restart: on-failure
+      depends_on:
+        db:
+          condition: service_healthy
+      healthcheck:
+        test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u admin:'myawesomegeoserver' http://localhost:8080/geoserver/rest/about/version.xml"
+        interval: 1m30s
+        timeout: 10s
+        retries: 3
+   node:
+      image: 'kartoza/geoserver:${TAG:-manual-build}'
+      volumes:
+         - geoserver-cluster-data:/opt/geoserver/data_dir
+         - ./tests:/tests
+      environment:
+        - BROKER_URL=tcp://master:61661
+        - READONLY=disabled
+        - CLUSTER_DURABILITY=false
+        - CLUSTERING=True
+        - TOGGLE_MASTER=true
+        - TOGGLE_SLAVE=true
+        - EMBEDDED_BROKER=disabled
+        - HOST=db
+        - POSTGRES_PORT=5432
+        - POSTGRES_DB=gis
+        - POSTGRES_USER=docker
+        - POSTGRES_PASS=docker
+        - SSL_MODE=allow
+        - GEOSERVER_ADMIN_PASSWORD=myawesomegeoserver
+        - GEOSERVER_ADMIN_USER=admin
+        - TEST_CLASS=test_clustering_node.GeoServerClusteringNode
+      restart: on-failure
+      depends_on:
+        master:
+          condition: service_started
+        db:
+          condition: service_healthy
+      healthcheck:
+        test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u admin:'myawesomegeoserver' http://localhost:8080/geoserver/rest/about/version.xml"
+        interval: 1m30s
+        timeout: 10s
+        retries: 3
+
+
+

scenario_tests/clustering/test.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# exit immediately if test fails
+set -e
+
+source ../test-env.sh
+
+# Run service
+if [[ $(dpkg -l | grep "docker-compose") > /dev/null ]];then
+    VERSION='docker-compose'
+  else
+    VERSION='docker compose'
+fi
+
+${VERSION} -f docker-compose.yml up -d
+
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  ${VERSION} -f docker-compose.yml logs -f &
+fi
+
+sleep 60
+
+# Test Master
+services=("master")
+
+for service in "${services[@]}"; do
+
+  # Execute tests
+  sleep 60
+  echo "Execute test for $service"
+  ${VERSION} -f docker-compose.yml exec "${service}" /bin/bash /tests/test.sh
+
+done
+
+# Test Node
+services=("node")
+
+for service in "${services[@]}"; do
+
+  # Execute tests
+  sleep 60
+  echo "Execute test for $service"
+  ${VERSION} -f docker-compose.yml exec "${service}" /bin/bash /tests/test.sh
+
+done
+
+${VERSION} -f docker-compose.yml down -v

scenario_tests/clustering/tests/init.sql

@@ -0,0 +1,20 @@
+ CREATE TABLE IF NOT EXISTS states (
+                    id integer not null
+                        constraint pkey primary key,
+                    geom geometry(Point, 4326),
+                    name varchar(30),
+                    alias varchar(30),
+                    description varchar(255)
+                );
+
+INSERT INTO states (id, geom, name, alias, description)
+                VALUES 
+                (
+                    1,
+                    st_setsrid(st_point(107.6097, 6.9120), 4326),
+                    'Bandung',
+                    'Paris van Java',
+                    'Asia-Africa conference was held here'                     
+                ) ON CONFLICT DO NOTHING;
+
+

scenario_tests/clustering/tests/test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+source /scripts/env-data.sh
+
+# execute tests
+pushd /tests
+
+cat << EOF
+Settings used:
+
+GEOSERVER_ADMIN_PASSWORD: ${GEOSERVER_ADMIN_PASSWORD}
+GEOSERVER_ADMIN_USER: ${GEOSERVER_ADMIN_USER}
+EOF
+
+python3 -m unittest -v ${TEST_CLASS}

scenario_tests/clustering/tests/test_clustering_master.py

@@ -0,0 +1,98 @@
+import unittest
+from os import environ
+
+from geo.Geoserver import Geoserver
+from requests import get, post, exceptions
+from requests.auth import HTTPBasicAuth
+
+
+class GeoServerClusteringMaster(unittest.TestCase):
+
+    def setUp(self):
+        # Define the GeoServer URL
+        self.gs_url = 'http://localhost:8080/geoserver'
+        # Define the PostGIS JNDI store name
+        self.store_name = 'gis'
+        self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin')
+        self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver')
+        self.geo_workspace_name = 'demo'
+
+    def test_publish_store(self):
+        geo = Geoserver(self.gs_url, username='%s' % self.geo_username, password='%s' % self.geo_password)
+        auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password)
+        # create workspace
+        try:
+            rest_url = '%s/rest/workspaces/%s.json' % (self.gs_url, self.geo_workspace_name)
+            response = get(rest_url, auth=auth)
+            response.raise_for_status()
+        except exceptions.HTTPError:
+            geo.create_workspace(workspace='%s' % self.geo_workspace_name)
+            geo.set_default_workspace('%s' % self.geo_workspace_name)
+
+        # Create the XML payload for the JNDI store configuration
+        xml = """
+        <dataStore>
+          <name>{name}</name>
+          <type>PostGIS</type>
+          <enabled>true</enabled>
+          <connectionParameters>
+            <entry key="host">db</entry>
+            <entry key="port">5432</entry>
+            <entry key="database">gis</entry>
+            <entry key="user">docker</entry>
+            <entry key="passwd">docker</entry>
+            <entry key="schema">public</entry>
+            <entry key="Estimated extends">true</entry>
+            <entry key="fetch size">1000</entry>
+            <entry key="encode functions">true</entry>
+            <entry key="Expose primary keys">false</entry>
+            <entry key="Support on the fly geometry simplification">true</entry>
+            <entry key="Batch insert size">1</entry>
+            <entry key="preparedStatements">false</entry>
+            <entry key="Method used to simplify geometries">FAST</entry>
+            <entry key="dbtype">postgis</entry>
+            <entry key="Loose bbox">true</entry>
+          </connectionParameters>
+          <disableOnConnFailure>false</disableOnConnFailure>
+        </dataStore>
+        """.format(name=self.store_name)
+
+        # Publish the store
+        response = post(self.gs_url + '/rest/workspaces/%s/datastores' % self.geo_workspace_name, auth=auth,
+                        headers={'Content-type': 'text/xml'},
+                        data=xml)
+
+        # Check that the response has a status code of 201 (Created)
+        self.assertEqual(response.status_code, 201)
+
+        # Check that the store exists
+        data_source_url = '%s/rest/workspaces/%s/datastores/%s.json' % (
+            self.gs_url, self.geo_workspace_name, self.store_name)
+        response = get(data_source_url, auth=auth)
+
+        # Check that the response has a status code of 200 (OK)
+        self.assertEqual(response.status_code, 200)
+
+        # Publish layer into GeoServer
+        try:
+            rest_url = '%s/rest/workspaces/%s/datastores/%s/featuretypes/states.json' % (
+                self.gs_url, self.geo_workspace_name, self.store_name)
+            publish_response = get(rest_url, auth=auth)
+            publish_response.raise_for_status()
+        except exceptions.HTTPError:
+            geo.publish_featurestore(workspace='%s'
+                                               % self.geo_workspace_name, store_name='%s' % self.store_name,
+                                     pg_table='states')
+        self.assertEqual(response.status_code, 200)
+
+        # Check that the layer exists
+        layer_url = '%s/rest/workspaces/%s/layers/states.json' % (
+            self.gs_url, self.geo_workspace_name)
+        response = get(layer_url, auth=auth)
+
+        # Check that the response has a status code of 200 (OK)
+        self.assertEqual(response.status_code, 200)
+
+
+if __name__ == '__main__':
+    unittest.main()

scenario_tests/clustering/tests/test_clustering_node.py

@@ -0,0 +1,53 @@
+import unittest
+from os import environ
+from requests import get, exceptions
+from requests.auth import HTTPBasicAuth
+
+
+class GeoServerClusteringNode(unittest.TestCase):
+
+    def setUp(self):
+        # Define the GeoServer URL
+        self.gs_url = 'http://localhost:8080/geoserver'
+        # Define the PostGIS store name
+        self.store_name = 'gis'
+        self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin')
+        self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver')
+        self.geo_workspace_name = 'demo'
+
+    def check_workspace_exists(self, auth):
+        rest_url = '%s/rest/workspaces/%s.json' % (self.gs_url, self.geo_workspace_name)
+        response = get(rest_url, auth=auth)
+        return response.status_code == 200
+
+    def check_data_store_exists(self, auth):
+        data_source_url = '%s/rest/workspaces/%s/datastores/%s.json' % (
+            self.gs_url, self.geo_workspace_name, self.store_name)
+        response = get(data_source_url, auth=auth)
+        return response.status_code == 200
+
+    def check_layer_exists(self, auth, layer_name):
+        layer_url = '%s/rest/workspaces/%s/layers/%s.json' % (
+            self.gs_url, self.geo_workspace_name, layer_name)
+        response = get(layer_url, auth=auth)
+        return response.status_code == 200
+
+    def test_workspace_exists(self):
+        auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password)
+        workspace_exists = self.check_workspace_exists(auth)
+        self.assertTrue(workspace_exists, "Workspace does not exist")
+
+    def test_data_store_exists(self):
+        auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password)
+        data_store_exists = self.check_data_store_exists(auth)
+        self.assertTrue(data_store_exists, "Data store does not exist")
+
+    def test_layer_exists(self):
+        auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password)
+        layer_name = 'states'
+        layer_exists = self.check_layer_exists(auth, layer_name)
+        self.assertTrue(layer_exists, "Layer does not exist")
+
+
+if __name__ == '__main__':
+    unittest.main()