diff --git a/readme.md b/readme.md index c00942f..a3c105e 100644 --- a/readme.md +++ b/readme.md @@ -66,3 +66,10 @@ Here's description of how to connect remote debugger to running Elasticsearch se 1. You should now see name of created configuration in upper right corner of IntelliJ IDEA. ![remote debug](images/run_debugging.png) 1. Click on a bug icon next to it to start debugger. 1. After successful connection, window like that should pop up from the bottom![successful debug connection](images/connected_debugger.png) + +## Running POC +1. Get a ReadonlyREST trial PRO license from [customer portal](https://readonlyrest.com/customer) +3. `cd ror-demo-cluster` +4. Run `bash -c "export ROR_ACTIVATION_KEY= && ./run.sh"` and select Kibana/es 7.17.21 (or 8.14.3 if you want to verify Kibana 8.x) +5. When everything is running add initial data by the `/bin/bash ./initData-7.x.sh` for Kibana 7.x or `/bin/bash ./initData-8.x.sh` for Kibana 8.x +6. You can access POC HTML with a simple Discover and Dashboard page for user1 here: http://localhost:18000 diff --git a/ror-cluster-elastic-cloud-demo/docker-compose.yml b/ror-cluster-elastic-cloud-demo/docker-compose.yml index aee4511..9d456f4 100644 --- a/ror-cluster-elastic-cloud-demo/docker-compose.yml +++ b/ror-cluster-elastic-cloud-demo/docker-compose.yml @@ -25,7 +25,7 @@ services: - ES_CLOUD_PROXY_ADDRESS=$ES_CLOUD_PROXY_ADDRESS - ES_CLOUD_SERVER_NAME=$ES_CLOUD_SERVER_NAME healthcheck: - test: [ "CMD", "curl", "-fk", "-u", "admin:admin", "http://localhost:9200/_cluster/health" ] + test: [ "CMD", "curl", "-fk", "-u", "admin:admin", "https://localhost:9200/_cluster/health" ] interval: 10s timeout: 10s retries: 30 @@ -53,7 +53,7 @@ services: - "15601:5601" environment: ELASTIC_USER_PASSWORD: elastic - ES_API_URL: http://es-ror:9200 + ES_API_URL: https://es-ror:9200 networks: - es-ror-network ulimits: diff --git a/ror-demo-cluster/clean.sh b/ror-demo-cluster/clean.sh index 1c33886..23baf04 100755 --- a/ror-demo-cluster/clean.sh +++ b/ror-demo-cluster/clean.sh @@ -1,3 +1,3 @@ #!/bin/bash -e -docker-compose rm --stop --force \ No newline at end of file +docker compose rm --stop --force diff --git a/ror-demo-cluster/conf/elasticsearch.yml b/ror-demo-cluster/conf/elasticsearch.yml deleted file mode 100644 index b8b51f0..0000000 --- a/ror-demo-cluster/conf/elasticsearch.yml +++ /dev/null @@ -1,5 +0,0 @@ -cluster.name: ror-cluster -node.name: ror-es01 -network.host: 0.0.0.0 - -xpack.security.enabled: false \ No newline at end of file diff --git a/ror-demo-cluster/conf/readonlyrest.yml b/ror-demo-cluster/conf/readonlyrest.yml deleted file mode 100644 index 360512a..0000000 --- a/ror-demo-cluster/conf/readonlyrest.yml +++ /dev/null @@ -1,22 +0,0 @@ -readonlyrest: - - access_control_rules: - - - name: "KIBANA" - type: allow - auth_key: kibana:kibana - verbosity: error - - - name: "ADMIN" - type: allow - verbosity: error - auth_key: admin:admin - kibana_access: admin - - - name: "User 1" - type: allow - verbosity: error - auth_key: "user1:test" - indices: [".kibana*", "my*"] - kibana_access: ro - kibana_index: '.kibana' diff --git a/ror-demo-cluster/conf/ror-cluster/elastic-certificates.p12 b/ror-demo-cluster/conf/ror-cluster/elastic-certificates.p12 new file mode 100644 index 0000000..1da043e Binary files /dev/null and b/ror-demo-cluster/conf/ror-cluster/elastic-certificates.p12 differ diff --git a/ror-demo-cluster/conf/ror-cluster/elasticsearch.yml b/ror-demo-cluster/conf/ror-cluster/elasticsearch.yml new file mode 100644 index 0000000..48b11fd --- /dev/null +++ b/ror-demo-cluster/conf/ror-cluster/elasticsearch.yml @@ -0,0 +1,19 @@ +cluster.name: ror-es-cluster +network.host: 0.0.0.0 + +cluster.remote.my-xpack-cluster.mode: sniff +cluster.remote.my-xpack-cluster.seeds: [ "es-xpack:9300" ] + +xpack.security.enabled: true + +xpack.security.http.ssl.enabled: true +xpack.security.http.ssl.verification_mode: none +xpack.security.http.ssl.client_authentication: none +xpack.security.http.ssl.keystore.path: elastic-certificates.p12 +xpack.security.http.ssl.truststore.path: elastic-certificates.p12 + +xpack.security.transport.ssl.enabled: true +xpack.security.transport.ssl.verification_mode: none +xpack.security.transport.ssl.client_authentication: none +xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 +xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 diff --git a/ror-demo-cluster/conf/kibana.yml b/ror-demo-cluster/conf/ror-cluster/kibana.yml similarity index 71% rename from ror-demo-cluster/conf/kibana.yml rename to ror-demo-cluster/conf/ror-cluster/kibana.yml index 1cd5143..16b1a28 100644 --- a/ror-demo-cluster/conf/kibana.yml +++ b/ror-demo-cluster/conf/ror-cluster/kibana.yml @@ -4,8 +4,11 @@ server.host: 0.0.0.0 elasticsearch.hosts: [ "${ES_API_URL}" ] elasticsearch.username: kibana elasticsearch.password: kibana +elasticsearch.ssl.verificationMode: none monitoring.ui.container.elasticsearch.enabled: true readonlyrest_kbn.logLevel: info readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' +readonlyrest_kbn.cookies.secure: true +readonlyrest_kbn.cookies.sameSite: 'none' \ No newline at end of file diff --git a/ror-demo-cluster/conf/ror-cluster/log4j2.properties b/ror-demo-cluster/conf/ror-cluster/log4j2.properties new file mode 100644 index 0000000..79c5867 --- /dev/null +++ b/ror-demo-cluster/conf/ror-cluster/log4j2.properties @@ -0,0 +1,88 @@ +# +# This file is part of ReadonlyREST. +# +# ReadonlyREST is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ReadonlyREST is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ +# +# +status=error +# log actionPost execution errors for easier debugging +logger.action.name=org.elasticsearch.action +logger.action.level=info +appender.console.type=Console +appender.console.name=console +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=[%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n +appender.rolling.type=RollingFile +appender.rolling.name=rolling +appender.rolling.fileName=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling.layout.type=PatternLayout +appender.rolling.layout.pattern=[%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.rolling.filePattern=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type=Policies +appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval=1 +appender.rolling.policies.time.modulate=true +rootLogger.level=info +rootLogger.appenderRef.console.ref=console +rootLogger.appenderRef.rolling.ref=rolling +appender.deprecation_rolling.type=RollingFile +appender.deprecation_rolling.name=deprecation_rolling +appender.deprecation_rolling.fileName=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +appender.deprecation_rolling.layout.type=PatternLayout +appender.deprecation_rolling.layout.pattern=[%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.deprecation_rolling.filePattern=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz +appender.deprecation_rolling.policies.type=Policies +appender.deprecation_rolling.policies.size.type=SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size=1GB +appender.deprecation_rolling.strategy.type=DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max=4 +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = deprecation +logger.deprecation.appenderRef.header_warning.ref = header_warning +logger.deprecation.appenderRef.deprecation_rolling.ref=deprecation_rolling +logger.deprecation.additivity=false +appender.index_search_slowlog_rolling.type=RollingFile +appender.index_search_slowlog_rolling.name=index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log +appender.index_search_slowlog_rolling.layout.type=PatternLayout +appender.index_search_slowlog_rolling.layout.pattern=[%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_search_slowlog_rolling.filePattern=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log +appender.index_search_slowlog_rolling.policies.type=Policies +appender.index_search_slowlog_rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval=1 +appender.index_search_slowlog_rolling.policies.time.modulate=true +logger.index_search_slowlog_rolling.name=index.search.slowlog +logger.index_search_slowlog_rolling.level=trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref=index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity=false +appender.index_indexing_slowlog_rolling.type=RollingFile +appender.index_indexing_slowlog_rolling.name=index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling.layout.type=PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern=[%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_indexing_slowlog_rolling.filePattern=${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +appender.index_indexing_slowlog_rolling.policies.type=Policies +appender.index_indexing_slowlog_rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval=1 +appender.index_indexing_slowlog_rolling.policies.time.modulate=true +logger.index_indexing_slowlog.name=index.indexing.slowlog.index +logger.index_indexing_slowlog.level=trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref=index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity=false + +appender.header_warning.type = HeaderWarningAppender +appender.header_warning.name = header_warning + +logger.ror.name=tech.beshu.ror +logger.ror.level=info diff --git a/ror-demo-cluster/conf/ror-cluster/readonlyrest.yml b/ror-demo-cluster/conf/ror-cluster/readonlyrest.yml new file mode 100644 index 0000000..29a0a66 --- /dev/null +++ b/ror-demo-cluster/conf/ror-cluster/readonlyrest.yml @@ -0,0 +1,37 @@ +readonlyrest: + + access_control_rules: + + - name: "::KIBANA::" + verbosity: error + type: allow + auth_key: kibana:kibana + + - name: "::ADMIN::" + verbosity: error + type: allow + auth_key: admin:admin + kibana: + access: admin + + # Allow JWT users to read all system Kibana indices without DLS, otherwise Kibana breaks + - name: "::UNFILTERED SYSTEM INDICES FOR USR*::" + indices: [ ".kibana*" ] # All kibana-related indices will be added implicitly (because of the presence of the kibana rule in the same block) + kibana: + access: rw + jwt_auth: + name: "jwt_provider_1" + + # Allow ANY jwt user to read his own data + - name: "::USER SEGMENTATION::" + users: [ "user*" ] # Optional, if you want to restrict users by name prefix, suffix, etc. + # Can filter also by group name, any string within the JWT claims! Here I am using the username + filter: '{"bool": { "must": { "match": { "username": "@{acl:user}" }}}}' + jwt_auth: + name: "jwt_provider_1" + + jwt: + - name: jwt_provider_1 + signature_algo: HMAC # can be NONE, RSA, HMAC (default), and EC + signature_key: "608c76e4bf65bb25c33369119594bad57273cefa13d463d788d487af89f183f8" + user_claim: name diff --git a/ror-demo-cluster/conf/xpack-cluster/elastic-certificates.p12 b/ror-demo-cluster/conf/xpack-cluster/elastic-certificates.p12 new file mode 100644 index 0000000..1da043e Binary files /dev/null and b/ror-demo-cluster/conf/xpack-cluster/elastic-certificates.p12 differ diff --git a/ror-demo-cluster/conf/xpack-cluster/elasticsearch.yml b/ror-demo-cluster/conf/xpack-cluster/elasticsearch.yml new file mode 100644 index 0000000..42b4f6d --- /dev/null +++ b/ror-demo-cluster/conf/xpack-cluster/elasticsearch.yml @@ -0,0 +1,16 @@ +cluster.name: xpack-es-cluster +network.host: 0.0.0.0 + +xpack.security.enabled: true + +xpack.security.http.ssl.enabled: true +xpack.security.http.ssl.verification_mode: none +xpack.security.http.ssl.client_authentication: none +xpack.security.http.ssl.keystore.path: elastic-certificates.p12 +xpack.security.http.ssl.truststore.path: elastic-certificates.p12 + +xpack.security.transport.ssl.enabled: true +xpack.security.transport.ssl.verification_mode: none +xpack.security.transport.ssl.client_authentication: none +xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 +xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 \ No newline at end of file diff --git a/ror-demo-cluster/conf/xpack-cluster/kibana.yml b/ror-demo-cluster/conf/xpack-cluster/kibana.yml new file mode 100644 index 0000000..365fa88 --- /dev/null +++ b/ror-demo-cluster/conf/xpack-cluster/kibana.yml @@ -0,0 +1,9 @@ +server.name: kibana-xpack +server.host: 0.0.0.0 + +elasticsearch.hosts: [ "${ES_API_URL}" ] +elasticsearch.username: kibana_system +elasticsearch.password: elastic +elasticsearch.ssl.verificationMode: none + +monitoring.ui.container.elasticsearch.enabled: true \ No newline at end of file diff --git a/ror-demo-cluster/conf/log4j2.properties b/ror-demo-cluster/conf/xpack-cluster/log4j2.properties similarity index 99% rename from ror-demo-cluster/conf/log4j2.properties rename to ror-demo-cluster/conf/xpack-cluster/log4j2.properties index c0daf0a..6486981 100644 --- a/ror-demo-cluster/conf/log4j2.properties +++ b/ror-demo-cluster/conf/xpack-cluster/log4j2.properties @@ -18,7 +18,7 @@ status=error # log actionPost execution errors for easier debugging logger.action.name=org.elasticsearch.action -logger.action.level=info +logger.action.level=debug appender.console.type=Console appender.console.name=console appender.console.layout.type=PatternLayout @@ -33,7 +33,7 @@ appender.rolling.policies.type=Policies appender.rolling.policies.time.type=TimeBasedTriggeringPolicy appender.rolling.policies.time.interval=1 appender.rolling.policies.time.modulate=true -rootLogger.level=info +rootLogger.level=debug rootLogger.appenderRef.console.ref=console rootLogger.appenderRef.rolling.ref=rolling appender.deprecation_rolling.type=RollingFile diff --git a/ror-demo-cluster/docker-compose.yml b/ror-demo-cluster/docker-compose.yml index ea0300f..5521acc 100644 --- a/ror-demo-cluster/docker-compose.yml +++ b/ror-demo-cluster/docker-compose.yml @@ -1,14 +1,16 @@ -version: "3.8" services: es-ror: build: context: . - dockerfile: images/es/${ES_DOCKERFILE} + dockerfile: images/ror-cluster/es/${ES_DOCKERFILE} args: ES_VERSION: $ES_VERSION ROR_VERSION: $ROR_ES_VERSION ROR_FILE: $ES_ROR_FILE + depends_on: + es-xpack: + condition: service_healthy ports: - "19200:9200" - "19300:9300" @@ -18,25 +20,28 @@ services: - node.name=es-ror-single - discovery.type=single-node - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000" - ES_VERSION=$ES_VERSION healthcheck: - test: [ "CMD", "curl", "-fk", "-u", "admin:admin", "http://localhost:9200/_cluster/health" ] + test: [ "CMD", "curl", "-fk", "-u", "kibana:kibana", "https://localhost:9200/_cluster/health" ] interval: 10s timeout: 10s retries: 30 start_period: 60s networks: - - es-ror-network + - ror-poc-network ulimits: memlock: soft: -1 hard: -1 + deploy: + resources: + limits: + memory: '768M' kbn-ror: build: context: . - dockerfile: images/kbn/${KBN_DOCKERFILE} + dockerfile: images/ror-cluster/kbn/${KBN_DOCKERFILE} args: KBN_VERSION: $KBN_VERSION ROR_VERSION: $ROR_KBN_VERSION @@ -49,14 +54,105 @@ services: - "15601:5601" environment: ELASTIC_USER_PASSWORD: elastic - ES_API_URL: http://es-ror:9200 + ES_API_URL: https://es-ror:9200 + healthcheck: + test: [ "CMD", "curl", "--fail", "http://localhost:5601/api/status" ] + interval: 10s + timeout: 10s + retries: 30 + start_period: 60s + networks: + - ror-poc-network + ulimits: + memlock: + soft: -1 + hard: -1 + deploy: + resources: + limits: + memory: '768M' + + web-server: + image: nginx:alpine + depends_on: + es-ror: + condition: service_healthy + kbn-ror: + condition: service_healthy + ports: + - "18000:80" + volumes: + - ./ror_poc_iframe.html:/usr/share/nginx/html/index.html + healthcheck: + test: [ "CMD", "curl", "--fail", "http://localhost:80" ] + interval: 10s + timeout: 10s + retries: 30 + start_period: 30s + networks: + - ror-poc-network + deploy: + resources: + limits: + memory: '128M' + + es-xpack: + build: + context: . + dockerfile: images/xpack-cluster/es/Dockerfile + args: + ES_VERSION: $ES_VERSION + ports: + - "29200:9200" + - "29300:9300" + - "5005:5000" + environment: + - cluster.name=xpack-es-cluster + - node.name=es-xpack-single + - discovery.type=single-node + - bootstrap.memory_lock=true + - ES_VERSION=$ES_VERSION + healthcheck: + test: [ "CMD", "curl", "-fk", "-u", "elastic:elastic", "https://localhost:9200/_security/user/kibana_system/_password", "-XPOST", "-H", "Content-Type: application/json", "-d", "{\"password\": \"elastic\"}" ] + interval: 10s + timeout: 10s + retries: 30 + start_period: 60s + networks: + - ror-poc-network + ulimits: + memlock: + soft: -1 + hard: -1 + deploy: + resources: + limits: + memory: '768M' + + kbn-xpack: + build: + context: . + dockerfile: images/xpack-cluster/kbn/Dockerfile + args: + KBN_VERSION: $KBN_VERSION + depends_on: + es-xpack: + condition: service_healthy + ports: + - "25601:5601" + environment: + ES_API_URL: https://es-xpack:9200 networks: - - es-ror-network + - ror-poc-network ulimits: memlock: soft: -1 hard: -1 + deploy: + resources: + limits: + memory: '768M' networks: - es-ror-network: + ror-poc-network: driver: bridge diff --git a/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-api b/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-api deleted file mode 100644 index 37e40af..0000000 --- a/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-api +++ /dev/null @@ -1,16 +0,0 @@ -ARG ES_VERSION - -FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - -ARG ES_VERSION -ARG ROR_VERSION - -COPY images/es/install-ror-es-using-api.sh /tmp/install-ror.sh - -USER root -RUN /tmp/install-ror.sh - -USER elasticsearch -COPY conf/readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml -COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml -COPY conf/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties diff --git a/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-file b/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-file deleted file mode 100644 index 137f946..0000000 --- a/ror-demo-cluster/images/es/Dockerfile-use-ror-binaries-from-file +++ /dev/null @@ -1,17 +0,0 @@ -ARG ES_VERSION - -FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} - -ARG ES_VERSION -ARG ROR_FILE - -COPY images/es/install-ror-es-using-file.sh /tmp/install-ror.sh -COPY $ROR_FILE /tmp/ror.zip - -USER root -RUN /tmp/install-ror.sh - -USER elasticsearch -COPY conf/readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml -COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml -COPY conf/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties diff --git a/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-api b/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-api new file mode 100644 index 0000000..7288fd7 --- /dev/null +++ b/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-api @@ -0,0 +1,24 @@ +ARG ES_VERSION + +FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + +ARG ES_VERSION +ARG ROR_VERSION + +COPY images/ror-cluster/es/install-ror-es-using-api.sh /tmp/install-ror.sh + +USER root +RUN /tmp/install-ror.sh + +USER elasticsearch +COPY conf/ror-cluster/readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml +COPY conf/ror-cluster/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml +COPY conf/ror-cluster/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties +COPY conf/ror-cluster/elastic-certificates.p12 /usr/share/elasticsearch/config/elastic-certificates.p12 + +RUN echo "" | /usr/share/elasticsearch/bin/elasticsearch-keystore create &&\ + echo 'elastic' | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin bootstrap.password -f &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password \ No newline at end of file diff --git a/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-file b/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-file new file mode 100644 index 0000000..ecd3dd6 --- /dev/null +++ b/ror-demo-cluster/images/ror-cluster/es/Dockerfile-use-ror-binaries-from-file @@ -0,0 +1,25 @@ +ARG ES_VERSION + +FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + +ARG ES_VERSION +ARG ROR_FILE + +COPY images/ror-cluster/es/install-ror-es-using-file.sh /tmp/install-ror.sh +COPY $ROR_FILE /tmp/ror.zip + +USER root +RUN /tmp/install-ror.sh + +USER elasticsearch +COPY conf/ror-cluster/readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml +COPY conf/ror-cluster/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml +COPY conf/ror-cluster/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties +COPY conf/ror-cluster/elastic-certificates.p12 /usr/share/elasticsearch/config/elastic-certificates.p12 + +RUN echo "" | /usr/share/elasticsearch/bin/elasticsearch-keystore create &&\ + echo 'elastic' | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin bootstrap.password -f &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password \ No newline at end of file diff --git a/ror-demo-cluster/images/es/install-ror-es-using-api.sh b/ror-demo-cluster/images/ror-cluster/es/install-ror-es-using-api.sh similarity index 100% rename from ror-demo-cluster/images/es/install-ror-es-using-api.sh rename to ror-demo-cluster/images/ror-cluster/es/install-ror-es-using-api.sh diff --git a/ror-demo-cluster/images/es/install-ror-es-using-file.sh b/ror-demo-cluster/images/ror-cluster/es/install-ror-es-using-file.sh similarity index 100% rename from ror-demo-cluster/images/es/install-ror-es-using-file.sh rename to ror-demo-cluster/images/ror-cluster/es/install-ror-es-using-file.sh diff --git a/ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-api b/ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-api similarity index 64% rename from ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-api rename to ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-api index 8bba51c..a80f64f 100644 --- a/ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-api +++ b/ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-api @@ -6,8 +6,8 @@ ARG KBN_VERSION ARG ROR_VERSION ARG ROR_ACTIVATION_KEY -COPY conf/kibana.yml /usr/share/kibana/config/kibana.yml -COPY images/kbn/install-ror-kbn-using-api.sh /tmp/install-ror.sh +COPY conf/ror-cluster/kibana.yml /usr/share/kibana/config/kibana.yml +COPY images/ror-cluster/kbn/install-ror-kbn-using-api.sh /tmp/install-ror.sh ENV ROR_ACTIVATION_KEY=$ROR_ACTIVATION_KEY diff --git a/ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-file b/ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-file similarity index 66% rename from ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-file rename to ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-file index c764227..741f229 100644 --- a/ror-demo-cluster/images/kbn/Dockerfile-use-ror-binaries-from-file +++ b/ror-demo-cluster/images/ror-cluster/kbn/Dockerfile-use-ror-binaries-from-file @@ -6,8 +6,8 @@ ARG KBN_VERSION ARG ROR_FILE ARG ROR_ACTIVATION_KEY -COPY conf/kibana.yml /usr/share/kibana/config/kibana.yml -COPY images/kbn/install-ror-kbn-using-file.sh /tmp/install-ror.sh +COPY conf/ror-cluster/kibana.yml /usr/share/kibana/config/kibana.yml +COPY images/ror-cluster/kbn/install-ror-kbn-using-file.sh /tmp/install-ror.sh COPY $ROR_FILE /tmp/ror.zip ENV ROR_ACTIVATION_KEY=$ROR_ACTIVATION_KEY diff --git a/ror-demo-cluster/images/kbn/install-ror-kbn-using-api.sh b/ror-demo-cluster/images/ror-cluster/kbn/install-ror-kbn-using-api.sh similarity index 100% rename from ror-demo-cluster/images/kbn/install-ror-kbn-using-api.sh rename to ror-demo-cluster/images/ror-cluster/kbn/install-ror-kbn-using-api.sh diff --git a/ror-demo-cluster/images/kbn/install-ror-kbn-using-file.sh b/ror-demo-cluster/images/ror-cluster/kbn/install-ror-kbn-using-file.sh similarity index 100% rename from ror-demo-cluster/images/kbn/install-ror-kbn-using-file.sh rename to ror-demo-cluster/images/ror-cluster/kbn/install-ror-kbn-using-file.sh diff --git a/ror-demo-cluster/images/xpack-cluster/es/Dockerfile b/ror-demo-cluster/images/xpack-cluster/es/Dockerfile new file mode 100644 index 0000000..063fa89 --- /dev/null +++ b/ror-demo-cluster/images/xpack-cluster/es/Dockerfile @@ -0,0 +1,17 @@ +ARG ES_VERSION + +FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} + +ARG ES_VERSION + +USER elasticsearch +COPY conf/xpack-cluster/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml +COPY conf/xpack-cluster/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties +COPY conf/xpack-cluster/elastic-certificates.p12 /usr/share/elasticsearch/config/elastic-certificates.p12 + +RUN echo "" | /usr/share/elasticsearch/bin/elasticsearch-keystore create &&\ + echo 'elastic' | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin bootstrap.password -f &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password &&\ + printf 'readonlyrest' | /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password \ No newline at end of file diff --git a/ror-demo-cluster/images/xpack-cluster/kbn/Dockerfile b/ror-demo-cluster/images/xpack-cluster/kbn/Dockerfile new file mode 100644 index 0000000..37dc101 --- /dev/null +++ b/ror-demo-cluster/images/xpack-cluster/kbn/Dockerfile @@ -0,0 +1,11 @@ +ARG KBN_VERSION + +FROM docker.elastic.co/kibana/kibana:${KBN_VERSION} + +ARG KBN_VERSION + +USER root +RUN chown -R kibana:kibana /usr/share/kibana/config + +USER kibana +COPY conf/xpack-cluster/kibana.yml /usr/share/kibana/config/kibana.yml diff --git a/ror-demo-cluster/initData-7.x.sh b/ror-demo-cluster/initData-7.x.sh new file mode 100755 index 0000000..2d05457 --- /dev/null +++ b/ror-demo-cluster/initData-7.x.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + +# INIT DATA IN THE MAIN XPACK CLUSTER +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/1" -H "Content-type: application/json" -d '{"username": "user1"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/2" -H "Content-type: application/json" -d '{"username": "user1"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/3" -H "Content-type: application/json" -d '{"username": "user1"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/4" -H "Content-type: application/json" -d '{"username": "user2"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/5" -H "Content-type: application/json" -d '{"username": "user2"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/6" -H "Content-type: application/json" -d '{"username": "user2"}' + +# ADD INDEX PATTERNS AND VISUALIZATIONS IN ROR-SIDECAR CLUSTER + +# Define variables +KIBANA_URL="http://localhost:15601" +INDEX_PATTERN_TITLE="my-xpack-cluster:ror_poc_001" +INDEX_PATTERN_TIME_FIELD="@timestamp" +VISUALIZATION_TITLE="Username Keyword Visualization" +DASHBOARD_TITLE="ROR POC 001 Dashboard" +KIBANA_USER="kibana" +KIBANA_PASSWORD="kibana" + +# Create Index Pattern +INDEX_PATTERN_RESPONSE=$(curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "{ + \"attributes\": { + \"title\": \"$INDEX_PATTERN_TITLE\", + \"timeFieldName\": \"$INDEX_PATTERN_TIME_FIELD\" + } +}") + +INDEX_PATTERN_ID=$(echo $INDEX_PATTERN_RESPONSE | jq -r '.id') + +# Create Visualization +VISUALIZATION_RESPONSE=$(curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/lens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"attributes":{"title":"ROR POC Vizualization","description":"","visualizationType":"lnsXY","state":{"visualization":{"legend":{"isVisible":true,"position":"right"},"valueLabels":"hide","fittingFunction":"None","yLeftExtent":{"mode":"full"},"yRightExtent":{"mode":"full"},"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"preferredSeriesType":"bar_stacked","layers":[{"layerId":"dbf664d8-92b8-4d48-8735-27f59ddbfb55","accessors":["baee8dad-44c2-4569-aae3-f519ca362087"],"position":"top","seriesType":"bar_stacked","showGridlines":false,"layerType":"data","xAccessor":"a4dd85e5-e343-4e3e-bd90-1eb6d213ecca"}]},"query":{"query":"","language":"kuery"},"filters":[],"datasourceStates":{"indexpattern":{"layers":{"dbf664d8-92b8-4d48-8735-27f59ddbfb55":{"columns":{"a4dd85e5-e343-4e3e-bd90-1eb6d213ecca":{"label":"Top values of username.keyword","dataType":"string","operationType":"terms","scale":"ordinal","sourceField":"username.keyword","isBucketed":true,"params":{"size":5,"orderBy":{"type":"column","columnId":"baee8dad-44c2-4569-aae3-f519ca362087"},"orderDirection":"desc","otherBucket":true,"missingBucket":false}},"baee8dad-44c2-4569-aae3-f519ca362087":{"label":"Count of records","dataType":"number","operationType":"count","isBucketed":false,"scale":"ratio","sourceField":"Records"}},"columnOrder":["a4dd85e5-e343-4e3e-bd90-1eb6d213ecca","baee8dad-44c2-4569-aae3-f519ca362087"],"incompleteColumns":{}}}}}}},"references":[{"type":"index-pattern","id":"'$INDEX_PATTERN_ID'","name":"indexpattern-datasource-current-indexpattern"},{"type":"index-pattern","id":"'$INDEX_PATTERN_ID'","name":"indexpattern-datasource-layer-dbf664d8-92b8-4d48-8735-27f59ddbfb55"}]}') + +# Extract the visualization ID +VISUALIZATION_ID=$(echo $VISUALIZATION_RESPONSE | jq -r '.id') + + +# Create Dashboard +curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/dashboard/ror_poc?overwrite=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"attributes":{"title":"ROR POC Dashboard","hits":0,"description":"","panelsJSON":"[{\"version\":\"7.17.15\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"7aa7d817-921c-46c8-9b65-ab22353a4bc9\"},\"panelIndex\":\"7aa7d817-921c-46c8-9b65-ab22353a4bc9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7aa7d817-921c-46c8-9b65-ab22353a4bc9\"}]","optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","version":1,"timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"}},"references":[{"name":"7aa7d817-921c-46c8-9b65-ab22353a4bc9:panel_7aa7d817-921c-46c8-9b65-ab22353a4bc9","type":"lens","id":"'$VISUALIZATION_ID'"}]}' + + diff --git a/ror-demo-cluster/initData-8.x.sh b/ror-demo-cluster/initData-8.x.sh new file mode 100755 index 0000000..6d7457d --- /dev/null +++ b/ror-demo-cluster/initData-8.x.sh @@ -0,0 +1,46 @@ +#!/bin/bash -e + +# INIT DATA IN THE MAIN XPACK CLUSTER + +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/1" -H "Content-type: application/json" -d '{"username": "user1", "@timestamp": "'$TIMESTAMP'"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/2" -H "Content-type: application/json" -d '{"username": "user1", "@timestamp": "'$TIMESTAMP'"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/3" -H "Content-type: application/json" -d '{"username": "user1", "@timestamp": "'$TIMESTAMP'"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/4" -H "Content-type: application/json" -d '{"username": "user2", "@timestamp": "'$TIMESTAMP'"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/5" -H "Content-type: application/json" -d '{"username": "user2", "@timestamp": "'$TIMESTAMP'"}' +curl -vk -u elastic:elastic -XPUT "https://localhost:29200/ror_poc_001/_doc/6" -H "Content-type: application/json" -d '{"username": "user2", "@timestamp": "'$TIMESTAMP'"}' + + +# ADD INDEX PATTERNS AND VISUALIZATIONS IN ROR-SIDECAR CLUSTER + +# Define variables +KIBANA_URL="http://localhost:15601" +INDEX_PATTERN_TITLE="my-xpack-cluster:ror_poc_001" +INDEX_PATTERN_TIME_FIELD="@timestamp" +VISUALIZATION_TITLE="Username Keyword Visualization" +DASHBOARD_TITLE="ROR POC 001 Dashboard" +KIBANA_USER="kibana" +KIBANA_PASSWORD="kibana" + +# Create Index Pattern +INDEX_PATTERN_RESPONSE=$(curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "{ + \"attributes\": { + \"title\": \"$INDEX_PATTERN_TITLE\", + \"timeFieldName\": \"$INDEX_PATTERN_TIME_FIELD\" + } +}") + +INDEX_PATTERN_ID=$(echo $INDEX_PATTERN_RESPONSE | jq -r '.id') + +# Create Visualization +VISUALIZATION_RESPONSE=$(curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/lens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"attributes":{"title":"ROR POC Vizualization","description":"","visualizationType":"lnsXY","state":{"visualization":{"legend":{"isVisible":true,"position":"right"},"valueLabels":"hide","fittingFunction":"None","yLeftExtent":{"mode":"full"},"yRightExtent":{"mode":"full"},"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"preferredSeriesType":"bar_stacked","layers":[{"layerId":"dbf664d8-92b8-4d48-8735-27f59ddbfb55","accessors":["baee8dad-44c2-4569-aae3-f519ca362087"],"position":"top","seriesType":"bar_stacked","showGridlines":false,"layerType":"data","xAccessor":"a4dd85e5-e343-4e3e-bd90-1eb6d213ecca"}]},"query":{"query":"","language":"kuery"},"filters":[],"datasourceStates":{"indexpattern":{"layers":{"dbf664d8-92b8-4d48-8735-27f59ddbfb55":{"columns":{"a4dd85e5-e343-4e3e-bd90-1eb6d213ecca":{"label":"Top values of username.keyword","dataType":"string","operationType":"terms","scale":"ordinal","sourceField":"username.keyword","isBucketed":true,"params":{"size":5,"orderBy":{"type":"column","columnId":"baee8dad-44c2-4569-aae3-f519ca362087"},"orderDirection":"desc","otherBucket":true,"missingBucket":false}},"baee8dad-44c2-4569-aae3-f519ca362087":{"label":"Count of records","dataType":"number","operationType":"count","isBucketed":false,"scale":"ratio","sourceField":"username.keyword"}},"columnOrder":["a4dd85e5-e343-4e3e-bd90-1eb6d213ecca","baee8dad-44c2-4569-aae3-f519ca362087"],"incompleteColumns":{}}}}}}},"references":[{"type":"index-pattern","id":"'$INDEX_PATTERN_ID'","name":"indexpattern-datasource-current-indexpattern"},{"type":"index-pattern","id":"'$INDEX_PATTERN_ID'","name":"indexpattern-datasource-layer-dbf664d8-92b8-4d48-8735-27f59ddbfb55"}]}') + +# Extract the visualization ID +VISUALIZATION_ID=$(echo $VISUALIZATION_RESPONSE | jq -r '.id') + + +# Create Dashboard +curl -u "$KIBANA_USER:$KIBANA_PASSWORD" -X POST "$KIBANA_URL/api/saved_objects/dashboard/ror_poc?overwrite=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"attributes":{"title":"ROR POC Dashboard","hits":0,"description":"","panelsJSON":"[{\"version\":\"7.17.15\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"7aa7d817-921c-46c8-9b65-ab22353a4bc9\"},\"panelIndex\":\"7aa7d817-921c-46c8-9b65-ab22353a4bc9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7aa7d817-921c-46c8-9b65-ab22353a4bc9\"}]","optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","version":1,"timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"}},"references":[{"name":"7aa7d817-921c-46c8-9b65-ab22353a4bc9:panel_7aa7d817-921c-46c8-9b65-ab22353a4bc9","type":"lens","id":"'$VISUALIZATION_ID'"}]}' + + diff --git a/ror-demo-cluster/ror_poc_iframe.html b/ror-demo-cluster/ror_poc_iframe.html new file mode 100644 index 0000000..e8295c7 --- /dev/null +++ b/ror-demo-cluster/ror_poc_iframe.html @@ -0,0 +1,67 @@ + + + + ReadonlyREST POC + + + + + + + diff --git a/ror-demo-cluster/run.sh b/ror-demo-cluster/run.sh index af1b0f3..3d42f5a 100755 --- a/ror-demo-cluster/run.sh +++ b/ror-demo-cluster/run.sh @@ -1,10 +1,5 @@ #!/bin/bash -e -if ! command -v docker-compose > /dev/null; then - echo "The script require docker-compose to be installed on your machine." - exit 1 -fi - echo -e " _____ _ ____ _ _____ ______ _____ _______ @@ -20,8 +15,8 @@ source ../utils/collect-info-about-ror-es-kbn.sh echo "Starting Elasticsearch and Kibana with installed ROR plugins ..." -docker-compose up -d --build --remove-orphans --force-recreate -docker-compose logs -f > ror-cluster.log 2>&1 & +docker compose up -d --build --remove-orphans --force-recreate --wait +docker compose logs -f > ror-cluster.log 2>&1 & echo -e " *********************************************************************** @@ -31,4 +26,4 @@ echo -e " *********************************************************************** " -echo -e "You can access ROR KBN here: http://localhost:15601 (regular user: 'user1:test' or admin user: 'admin:admin')" +echo -e "You can access POC HTML with a simple Discover and Dashboard page for user1 here: http://localhost:18000"