Skip to content

docker compose sample(telegram)

Jump to bottom
Naoyuki Sano edited this page Apr 24, 2022 · 13 revisions

document

telegram_proxy_login and telegram_proxy_pass are not mentioned in the documentation
https://elastalert2.readthedocs.io/en/latest/ruletypes.html#telegram

telegram_bot_token and telegram_room_id are required

telegram_bot_token
telegram_room_id
telegram_api_url・・・default value「api.telegram.org」
telegram_proxy
telegram_proxy_login
telegram_proxy_pass

Praeco

Add telegram support https://github.com/johnsusek/praeco/commit/b75bda44109f9c03b54fac064b4c95abd81e344a

Telegram_room_id can be set on the web screen only

The following settings are set in BaseRule.config
https://github.com/johnsusek/praeco/blob/master/UPGRADING.md#v039---v040

telegram_api_url does not need to be set if there is no problem with the default value "api.telegram.org"

telegram_bot_token

telegram

Telegramのインストール方法

telegram bot can't send messages to super group

Telegram Bot - how to get a group chat id?

setting and Operation check

Praeco Web UI Setting

b

BaseRule.config

a

/home/user/dkwork2/es
|--Dockerfiles
|  |--Dockerfile.elastalert
|--docker-compose.yml
|--es
|  |--config
|  |  |--elasticsearch.yml
|  |--data
|--kibana
|  |--config
|  |  |--kibana.yml
|--praeco
|  |--bin
|  |  |--elastalert-start.sh
|  |  |--elastic_search_status.sh
|  |--config
|  |  |--api.config.json
|  |  |--elastalert.yaml
|  |--nginx_config
|  |  |--nginx.conf
|  |  |--default.conf
|  |--public
|  |  |--praeco.config.json
|  |--rule_templates
|  |--rules
|  |  |--BaseRule.config
|--fluentd
|  |--dockerfiles
|  |  |--Dockerfile
|  |--etc
|  |  |--fluent.conf
|--mariadb
|  |--data
|  |--etc
|  |  |--mymariadb.cnf
|  |--log
|  |  |--error.log
|  |  |--general.log
|  |  |--slow.log

Dockerfiles/Dockerfile.elastalert

FROM praecoapp/elastalert-server:latest

USER root

RUN apk update && \
    apk add bash curl && \
    rm -rf /var/cache/apk/*

ADD praeco/bin/elastalert-start.sh /usr/local/bin/
ADD praeco/bin/elastic_search_status.sh /usr/local/bin/

RUN chmod +x /usr/local/bin/elastalert-start.sh 
RUN chmod +x /usr/local/bin/elastic_search_status.sh

USER node

ENTRYPOINT ["/usr/local/bin/elastalert-start.sh"]

docker-compose.yml

version: "3.7"
services:
  elasticsearch:
    container_name: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
    restart: always
    volumes:
      - ./es/data:/usr/share/elasticsearch/data
      - ./es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:9200 || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 180s

  kibana:
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:7.17.1
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    restart: always
    volumes:
      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:5601/api/status || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 200s
 
  elastalert:
    container_name: elastalert
    build:
      context: .
      dockerfile: Dockerfiles/Dockerfile.elastalert
    image: elastalert-server:3.0.0
    ports:
      - 3030:3030
      - 3333:3333
    depends_on:
      - elasticsearch
    restart: always
    volumes:
      - ./praeco/config/elastalert.yaml:/opt/elastalert/config.yaml
      - ./praeco/config/api.config.json:/opt/elastalert-server/config/config.json
      - ./praeco/rules:/opt/elastalert/rules
      - ./praeco/rule_templates:/opt/elastalert/rule_templates
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:3030 || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 200s

  praeco:
    container_name: praeco
    image: praecoapp/praeco:latest
    ports:
      - 8080:8080
    depends_on:
      - elastalert
    restart: always
    volumes:
      - ./praeco/public/praeco.config.json:/var/www/html/praeco.config.json
      - ./praeco/nginx_config/nginx.conf:/etc/nginx/nginx.conf
      - ./praeco/nginx_config/default.conf:/etc/nginx/conf.d/default.conf
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:8080 || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 200s

  fluentd:
    container_name: fluentd
    build: ./fluentd/dockerfiles
    image: fluentd:1.14.6
    ports:
      - 24224:24224
      - 24224:24224/udp
    environment:
      - FLUENTD_CONF=fluent.conf
    volumes:
      - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf
      - ./mariadb/log:/var/log/mysql
    user: root
    restart: always

  mariadb:
    container_name: mariadb
    image: mariadb:10.4.12
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=mariadb
    volumes:
      - ./mariadb/etc:/etc/mysql/conf.d
      - ./mariadb/log:/var/log/mysql
    restart: always

es/config/elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0
#discovery.zen.minimum_master_nodes: 1

kibana/config/kibana.yml

server.name: kibana
server.host: "0.0.0.0"
server.shutdownTimeout: "5s"
elasticsearch.hosts: http://elasticsearch:9200
monitoring.ui.container.elasticsearch.enabled: true

praeco/bin/elastalert-start.sh

#!/bin/bash

set -e

echo "Giving Elasticsearch at $ELASTICSEARCH_URL time to start..."

elastic_search_status.sh

echo "Starting ElastAlert!"
npm start

praeco/bin/elastic_search_status.sh

#!/bin/bash

set -e

if [ $# -gt 0 ]; then
  ES_URL="$1"
elif [[ -n $ELASTICSEARCH_URL ]]; then
  ES_URL="$ELASTICSEARCH_URL"
elif [[ -n $ES_HOST ]] && [[ -n $ES_PORT ]]; then
  ES_URL="http://$ES_HOST:$ES_PORT"
else
  ES_URL="http://elasticsearch:9200"
fi

until [[ "$(curl -fsSL "$ES_URL/_cat/health?h=status" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" =~ ^(yellow|green)$ ]]; do
  # printf '+' >&2
  sleep 1
done

echo "Elasticsearch is up and healthy at "$ES_URL"" >&2

praeco/config/api.config.json

Default settings

{
  "appName": "elastalert-server",
  "port": 3030,
  "wsport": 3333,
  "elastalertPath": "/opt/elastalert",
  "verbose": false,
  "es_debug": false,
  "debug": false,
  "rulesPath": {
    "relative": true,
    "path": "/rules"
  },
  "templatesPath": {
    "relative": true,
    "path": "/rule_templates"
  },
  "es_host": "elasticsearch",
  "es_port": 9200,
  "es_username": "",
  "es_password": "",
  "es_ssl": false,
  "writeback_index": "praeco_elastalert_status"
}

praeco/config/elastalert.yaml

Default settings

# The elasticsearch hostname for metadata writeback
# Note that every rule can have its own elasticsearch host
es_host: elasticsearch

# The elasticsearch port
es_port: 9200

# This is the folder that contains the rule yaml files
# Any .yaml file will be loaded as a rule
rules_folder: rules

# How often ElastAlert will query elasticsearch
# The unit can be anything from weeks to seconds
run_every:
  seconds: 60

# ElastAlert will buffer results from the most recent
# period of time, in case some log sources are not in real time
buffer_time:
  minutes: 1

# Optional URL prefix for elasticsearch
#es_url_prefix: elasticsearch

# Connect with TLS to elasticsearch
#use_ssl: True

# Verify TLS certificates
#verify_certs: True

# GET request with body is the default option for Elasticsearch.
# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
# for details
#es_send_get_body_as: GET

# Option basic-auth username and password for elasticsearch
#es_username: someusername
#es_password: somepassword

# The index on es_host which is used for metadata storage
# This can be a unmapped index, but it is recommended that you run
# elastalert-create-index to set a mapping
writeback_index: praeco_elastalert_status

# If an alert fails for some reason, ElastAlert will retry
# sending the alert until this time period has elapsed
alert_time_limit:
  days: 2

skip_invalid: True

profile: default

praeco/nginx_config/nginx.conf

user  www-data;
worker_processes  1;

error_log  stderr warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    sendfile        off;

    keepalive_timeout  60;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

praeco/nginx_config/default.conf

# cache github api
proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=github_api_cache:60m max_size=10g 
                 inactive=60m use_temp_path=off;

server {
  listen 8080;
  #rewrite ^/my-path(/.*)$ $1 last;

  location /api {
      rewrite ^/api/?(.*)$ /$1 break;
      proxy_pass http://elastalert:3030/;
  }

  location /api-ws {
      rewrite ^/api-ws/?(.*)$ /$1 break;
      proxy_pass http://elastalert:3333/;
      proxy_http_version 1.1;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "Upgrade";
  }

  location /api-app/releases {
      proxy_cache github_api_cache;
      proxy_pass https://api.github.com/repos/ServerCentral/praeco/releases;
  }

  location / {
      root /var/www/html;
      try_files $uri $uri/ /index.html;
  }
}

praeco/public/praeco.config.json

Default settings

{
  "appUrl": "http://127.0.0.1:8080",
  "errorLoggerUrl": "",
  "hidePreconfiguredFields": []
}

praeco/rules/BaseRule.config

telegram_bot_token: 'xxxxxxxxxx:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

fluentd/dockerfiles/Dockerfile

FROM fluent/fluentd:v1.14.6-debian-1.0

# Use root account to use apt
USER root

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish
RUN buildDeps="sudo make gcc g++ libc-dev" \
 && apt-get update \
 && apt-get install -y --no-install-recommends $buildDeps \
 && sudo gem install fluent-plugin-mysqlslowquery -v 0.0.9 \
&& sudo gem install 'elasticsearch:7.17.0' \
                     'elasticsearch-api:7.17.0' \
                     'elasticsearch-xpack:7.17.0' \
                     fluent-plugin-elasticsearch \
                     oj \
                     fluent-plugin-rewrite-tag-filter \
 && sudo gem sources --clear-all \
 && SUDO_FORCE_REMOVE=yes \
    apt-get purge -y --auto-remove \
                  -o APT::AutoRemove::RecommendsImportant=false \
                  $buildDeps \
 && rm -rf /var/lib/apt/lists/* \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

USER fluent

fluentd/etc/fluent.conf

<source>
  @type mysql_slow_query
  path /var/log/mysql/slow.log
  pos_file /tmp/mysql/slow.pos
  tag mysqld.slow_query
   <parse>
     @type none
   </parse>
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/general.log
  pos_file /tmp/mysql/general.pos
  tag mysqld.general
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/error.log
  pos_file /tmp/mysql/error.pos
  tag mysqld.error
</source>

<match **.**>
  @type copy
  <store>
    @type stdout
  </store>
  <store>
    @type elasticsearch
    include_tag_key true
    tag_key @log_name
    logstash_format true
    logstash_prefix mariadb-log
    host elasticsearch
    port 9200
    index_name mysql
    flush_interval 10s
  </store>
</match>

mariadb/etc/mymariadb.cnf

[mysqld]
general_log
general_log_file=/var/log/mysql/general.log
slow_query_log
slow_query_log_file=/var/log/mysql/slow.log
long_query_time=5 
log-queries-not-using-indexes
log-error=/var/log/mysql/error.log

setting

cd /home/user/dkwork2/es
chmod 777 es/data
chmod 777 mariadb/data
chmod 777 mariadb/log
chmod -R 777 praeco/rules praeco/rule_templates
docker-compose up -d

a

Receive alert

telegram

Clone this wiki locally