Merge pull request #204 from kuskoman/v2

V2- YAML config, refactor, multi-client monitoring

.docker/exporter-config.yml

@@ -0,0 +1,9 @@
+logstash:
+  servers:
+    - url: "http://logstash:9600"
+    - url: "http://logstash2:9600"
+server:
+  host: "0.0.0.0"
+  port: 9198
+logging:
+  level: "debug"

.github/workflows/go-application.yml

@@ -157,7 +157,7 @@ jobs:
           VERSION: ${{ github.ref }}
 
       - name: Upload Linux binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logstash-exporter-linux
           path: out/main-linux
@@ -182,7 +182,7 @@ jobs:
           VERSION: ${{ github.ref }}
 
       - name: Upload Mac binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logstash-exporter-darwin
           path: out/main-darwin
@@ -207,7 +207,7 @@ jobs:
           VERSION: ${{ github.ref }}
 
       - name: Upload Windows binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logstash-exporter-windows
           path: out/main-windows
@@ -227,7 +227,7 @@ jobs:
         run: docker save logstash-exporter:latest | gzip > logstash-exporter.tar.gz
 
       - name: Upload Docker image as an artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logstash-exporter-docker-image
           path: logstash-exporter.tar.gz
@@ -297,7 +297,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Download Docker image
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: logstash-exporter-docker-image
           path: .helm/files
@@ -308,7 +308,7 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v3
         with:
-          version: 3.11.2
+          version: 3.13.3
 
       - name: Include current chart versions
         working-directory: .helm/logstash-integration-test-chart
@@ -379,7 +379,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Download binary
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: logstash-exporter-${{ matrix.binary }}
       - name: Generate sha256 checksum

.gitignore

@@ -42,3 +42,6 @@ _debug*.yaml
 
 # Readme generator files
 helm-generator/
+
+# Local config.yml
+config.yml

.helm/logstash-integration-test-chart/Chart.lock

@@ -4,12 +4,12 @@ dependencies:
   version: 8.5.1
 - name: apache
   repository: https://charts.bitnami.com/bitnami
-  version: 9.2.23
+  version: 10.2.4
 - name: prometheus
   repository: https://prometheus-community.github.io/helm-charts
-  version: 20.2.0
+  version: 25.8.2
 - name: logstash-exporter
   repository: file://../../chart/
-  version: v1.0.2
-digest: sha256:1733217c222c212aac4b58826c147d7acb2f61fe01ce6985b139050803915d92
-generated: "2023-04-12T10:38:00.905306965+02:00"
+  version: 1.0.2
+digest: sha256:7d4c4f09f7947734ed527834d32065bebdb655fddd854fe9ba0481d3d9939e7f
+generated: "2023-12-26T20:27:15.623329+01:00"

.helm/logstash-integration-test-chart/Chart.yaml

@@ -5,16 +5,16 @@ type: application
 version: 0.1.0
 dependencies:
   - name: logstash
-    version: "8.5.1"
+    version: "^8.5.1"
     repository: https://helm.elastic.co
     condition: logstash.enabled
   # we are replacing elasticsearch with apache for testing purposes
   - name: apache
-    version: "^9.2.23"
+    version: "^10.2.4"
     repository: https://charts.bitnami.com/bitnami
     condition: apache.enabled
   - name: prometheus
-    version: "^20.2.0"
+    version: "^25.8.2"
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus.enabled
   - name: logstash-exporter

.helm/logstash-integration-test-chart/values.yaml

@@ -23,6 +23,12 @@ logstash:
 
 prometheus:
   enabled: true
+  alertmanager:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  prometheus-pushgateway:
+    enabled: false
   service:
     type: ClusterIP
   config: |

Makefile

@@ -110,6 +110,10 @@ clean-elasticsearch:
 upgrade-dependencies:
 	go get -u ./...
 
+#: Migrates configuration from v1 to v2
+migrate-v1-to-v2:
+	./scripts/migrate_v1_to_v2.sh
+
 #: Shows info about available commands
 help:
 	@grep -B1 -E "^[a-zA-Z0-9_-]+\:([^\=]|$$)" Makefile \

README.md

@@ -8,6 +8,10 @@ The project was created as rewrite of existing awesome application
 which was also written in Go, but it was not maintained for a long time.
 A lot of code was reused from the original project.
 
+**Important:** V2 version of the application is currently in beta. It is recommended to use V1 version in production.
+It is still maintained and available under [v1 branch](https://github.com/kuskoman/logstash-exporter/tree/v1).
+Make sure to check the [Migration](#migration) section before upgrading to V2.
+
 **Important:** Because of limited workforces, this project is tested only against a single Logstash version.
 You can check the tested version in [docker-compose.yml](./docker-compose.yml) file.
 Using this exporter with other versions of Logstash may not work properly (although most of the metrics should work).
@@ -87,25 +91,56 @@ The Helm chart has its own [README](./chart/README.md).
 
 ### Configuration
 
-The application can be configured using the following environment variables, which are also loaded from `.env` file:
+The application is now configured using a YAML file instead of environment variables. An example configuration is as follows:
 
-| Variable Name  | Description                                                 | Default Value            |
-| -------------- | ----------------------------------------------------------- | -----------------------  |
-| `LOGSTASH_URL` | URL to Logstash API                                         | `http://localhost:9600`  |
-| `PORT`         | Port on which the application will be exposed               | `9198`                   |
-| `HOST`         | Host on which the application will be exposed               | empty string             |
-| `LOG_LEVEL`    | [Log level](https://pkg.go.dev/golang.org/x/exp/slog#Level) | empty (defaults to info) |
+```yaml
+logstash:
+  servers:
+    - url: "http://logstash:9600" # URL to Logstash API
+    - url: "http://logstash2:9600"
+server:
+  host: "0.0.0.0" # Host on which the application will be exposed (default: all interfaces)
+  port: 9198      # Port on which the application will be exposed
+logging:
+  level: "debug"  # Log level (debug, info, warn, error)
+```
 
 All configuration variables can be checked in the [config directory](./config/).
 
+Previously the application was configured using environment variables. The old configuration is no longer supported,
+however a [migration script](./scripts/migrate_env_to_yaml.sh) is provided to migrate the old configuration to the new one.
+See more in the [Migration](#migration) section.
+
+## Migration
+
+### From v1 to v2
+
+#### With Migration Script
+
+1. Backup Configuration: Save your existing .env file.
+2. Run Migration Script: Execute ./scripts/migrate_v1_to_v2.sh [path/to/.env]. If your .env is in the default location,
+    simply run ./scripts/migrate_v1_to_v2.sh.
+    You can use it via make: `make migrate-v1-to-v2`.
+3. Save New Configuration: Redirect output to new_config.yaml using ./scripts/migrate_v1_to_v2.sh.sh > config.yaml.
+4. Set `EXPORTER_CONFIG_LOCATION`: Update this environment variable to the path of config.yaml (or leave the default).
+5. Test Application: Ensure the application runs correctly with the new configuration.
+
+#### Without Migration Script
+
+1. Backup Configuration: Keep a record of your current environment variables.
+2. Create YAML File: Manually create a config.yaml file following the new format.
+3. Transfer Settings: Copy values from your .env file or noted environment variables into the corresponding sections in new_config.yaml.
+4. Set EXPORTER_CONFIG_LOCATION: Point this environment variable to new_config.yaml.
+5. Test Application: Check if the application functions properly with the new configuration.
+
 ## Building
 
 ### Makefile
 
 #### Available Commands
 
 <!--- GENERATED by ./scripts/add_descriptions_to_readme.sh --->
- 
+
 - `make all`: Builds binary executables for all OS (Win, Darwin, Linux).
 - `make run`: Runs the Go Exporter application.
 - `make build-linux`: Builds a binary executable for Linux.
@@ -127,7 +162,7 @@ All configuration variables can be checked in the [config directory](./config/).
 - `make helm-readme`: Generates Helm chart README.md file.
 - `make clean-elasticsearch`: Cleans Elasticsearch data, works only with default ES port. The command may take a very long time to complete.
 - `make help`: Shows info about available commands.
- 
+
 <!--- **************************************************** --->
 
 #### File Structure
@@ -138,91 +173,91 @@ The binary executables are saved in the out directory.
 #### Example Usage
 
 <!--- GENERATED by ./scripts/add_descriptions_to_readme.sh --->
- 
+
 Builds binary executables for all OS (Win, Darwin, Linux):
- 
+
     make all
- 
+
 Runs the Go Exporter application:
- 
+
     make run
- 
+
 Builds a binary executable for Linux:
- 
+
     make build-linux
- 
+
 Builds a binary executable for Darwin:
- 
+
     make build-darwin
- 
+
 Builds a binary executable for Windows:
- 
+
     make build-windows
- 
+
 Builds a Docker image for the Go Exporter application:
- 
+
     make build-docker
- 
+
 Builds a multi-arch Docker image (`amd64` and `arm64`):
- 
+
     make build-docker-multi
- 
+
 Deletes all binary executables in the out directory:
- 
+
     make clean
- 
+
 Runs all tests:
- 
+
     make test
- 
+
 Displays test coverage report:
- 
+
     make test-coverage
- 
+
 Starts a Docker-compose configuration:
- 
+
     make compose
- 
+
 Starts a Docker-compose configuration until it's ready:
- 
+
     make wait-for-compose
- 
+
 Stops a Docker-compose configuration:
- 
+
     make compose-down
- 
+
 Verifies the metrics from the Go Exporter application:
- 
+
     make verify-metrics
- 
+
 Pulls the Docker image from the registry:
- 
+
     make pull
- 
+
 Shows logs from the Docker-compose configuration:
- 
+
     make logs
- 
+
 Minifies the binary executables:
- 
+
     make minify
- 
+
 Installs readme-generator-for-helm tool:
- 
+
     make install-helm-readme
- 
+
 Generates Helm chart README.md file:
- 
+
     make helm-readme
- 
+
 Cleans Elasticsearch data, works only with default ES port. The command may take a very long time to complete:
- 
+
     make clean-elasticsearch
- 
+
 Shows info about available commands:
- 
+
     make help
- 
+
 <!--- **************************************************** --->
 
 ## Helper Scripts