Skip to content

Commit

Permalink
[Backport 2.x] Added HAProxy Integration (opensearch-project#1277)
Browse files Browse the repository at this point in the history
* Added HAProxy Integration

* Updated test directory  for HAProxy integration

* updated docker-compose.yaml and otel-converter.lua

---------

(cherry picked from commit f1237c1)

Signed-off-by: Pothula-Kalyani-ML <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
(cherry picked from commit 2a4c057)
  • Loading branch information
opensearch-trigger-bot[bot] authored and A9 Swift Project User committed Jan 8, 2024
1 parent 871cbc0 commit 37d574d
Show file tree
Hide file tree
Showing 20 changed files with 873 additions and 2 deletions.
4 changes: 2 additions & 2 deletions auto_sync_commit_metadata.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"last_github_commit": "e36239f2a74d0e4366475cca28b3857f6910e83a",
"last_gitfarm_commit": "e6d758faedd98842b0e875ac43c2f4662c8a91b6"
"last_github_commit": "2a4c05712a0002c8a965de90e10b652c96a8dfb5",
"last_gitfarm_commit": "eebdf09a2ca2b1cb1ed65bd222d3ed7cde0d821a"
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T06:46:14.029Z","observedTimestamp":"2023-11-20T06:46:14.029Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:06:46:14 +0000] \"GET /monitor HTTP/1.1\" 200 23741 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"300","bytes":23741},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:24.734Z","observedTimestamp":"2023-11-20T07:10:24.734Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:24 +0000] \"GET /monitor HTTP/1.1\" 200 23881 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"400","bytes":23881},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:19.628Z","observedTimestamp":"2023-11-20T07:10:19.628Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:19 +0000] \"GET /monitor HTTP/1.1\" 200 23881 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23881},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:14.505Z","observedTimestamp":"2023-11-20T07:10:14.505Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:14 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:09.302Z","observedTimestamp":"2023-11-20T07:10:09.302Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:09 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:04.151Z","observedTimestamp":null,"body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:04 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:04.151Z","observedTimestamp":"2023-11-20T07:10:04.000Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:04 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"404","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-21T08:57:33.000Z","observedTimestamp":"2023-11-21T08:57:43.000Z","body":"10.81.1.1:61108 stats stats/<NOSRV>-1/-1/-1/-1/10009a 1/1/0/0/00/0--cR--[21/Nov/2023:08:57:43 +0000] \" HTTP/1.1\" 408 0 \"\"<BADREQ>\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{},"response":{"status_code":"408","bytes":1},"url":"/monitor"}}]
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
{
"name": "haproxy",
"version": "1.0.0",
"displayName": "Haproxy Dashboard",
"description": "Haproxy logs collector",
"license": "Apache-2.0",
"type": "logs",
"labels": ["Observability", "Logs"],
"author": "OpenSearch",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info",
"statics": {
"logo": {
"annotation": "Haproxy Logo",
"path": "logo.svg"
},
"gallery": [
{
"annotation": "Haproxy Dashboard",
"path": "dashboard1.png"
},
{
"annotation": "Haproxy Dashboard view",
"path": "dashboard2.png"
}
]
},
"components": [
{
"name": "communication",
"version": "1.0.0"
},
{
"name": "http",
"version": "1.0.0"
},
{
"name": "logs",
"version": "1.0.0"
}
],
"assets": {
"savedObjects": {
"name": "haproxy",
"version": "1.0.0"
}
},
"sampleData": {
"path": "sample.json"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# HAProxy Integration

## What is HAProxy?

HAProxy is open-source software that provides a high availability load balancer and proxy server for TCP and HTTP-based applications.

See additional details [here](http://www.haproxy.org/).

## What is HAProxy Integration?

An integration is a bundle of pre-canned assets that are packaged together in a meaningful manner.
HAProxy integration includes dashboards, visualisations, queries and an index mapping.

### Dashboards

![](../static/dashboard1.png)
![](../static/dashboard2.png)

Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
version: '3'

services:
opensearch:
image: opensearchstaging/opensearch:3.0.0
container_name: opensearch
environment:
- cluster.name=opensearch-cluster # Name the cluster
- node.name=opensearch # Name the node that will run in this container
- discovery.seed_hosts=opensearch # Nodes to look for when discovering the cluster
- cluster.initial_cluster_manager_nodes=opensearch # Nodes eligibile to serve as cluster manager
- bootstrap.memory_lock=true # Disable JVM heap memory swapping
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
- "DISABLE_INSTALL_DEMO_CONFIG=true" # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
- "DISABLE_SECURITY_PLUGIN=true" # Disables security plugin
ulimits:
memlock:
soft: -1 # Set memlock to unlimited (no soft or hard limit)
hard: -1
nofile:
soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
hard: 65536
volumes:
- opensearch:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
ports:
- 9200:9200
- 9600:9600
expose:
- "9200"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"]
interval: 5s
timeout: 25s
retries: 4
networks:
- opensearch-net # All of the containers will join the same Docker bridge network
haproxy:
container_name: haproxy
image: haproxytech/haproxy-alpine:2.4
volumes:
# - ./haproxy-otel/opentelemetry_module.conf:/etc/nginx/conf.d/opentelemetry_module.conf
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- 8405:8405
depends_on:
- flask-app
- fluentbit
networks:
- opensearch-net # All of the containers will join the same Docker bridge network
links:
- fluentbit
logging:
driver: "fluentd"
options:
fluentd-address: 127.0.0.1:24224
tag: haproxy.access
fluentbit:
container_name: fluentbit
image: fluent/fluent-bit:latest
volumes:
- ./fluent-bit:/fluent-bit/etc
ports:
- "24224:24224"
- "24224:24224/udp"
depends_on:
- opensearch
networks:
- opensearch-net
redis:
image: redis
ports:
- 6357:6357
networks:
- opensearch-net
flask-app:
build: flask-app
ports:
- 5000:5000
depends_on:
- redis
volumes:
- ./flask-app/app.py:/code/app.py
networks:
- opensearch-net

volumes:
opensearch:

networks:
opensearch-net:
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM python:latest
ADD . /code
WORKDIR /code
RUN pip install -r requirements.txt

RUN mkdir -p /run/haproxy/
RUN export VALUE=$(id -u haproxy)

CMD ["flask", "run", "-h", "0.0.0.0", "-p", "5000", "--debug"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from random import random, randint
from flask import Flask
from redis import Redis

app = Flask(__name__)
redis = Redis(host='redis', port=6379)


@app.route("/")
def hits():
if random() < 0.20:
return "Random error", 500
h = redis.incr('hits')
return f"This page has been viewed {h} time(s)"

@app.route("/dice")
def roll_dice():
if random() < 0.05:
return "Random error", 500
return str(randint(1, 6))
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
flask==2.3.2
redis
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
[SERVICE]
Parsers_File parsers.conf

[INPUT]
Name forward
Port 24224

[FILTER]
Name parser
Match haproxy.access
Key_Name log
Parser haproxy

[FILTER]
Name lua
Match haproxy.access
Script otel-converter.lua
call convert_to_otel

[OUTPUT]
Name opensearch
Match haproxy.access
Host opensearch
Port 9200
Index ss4o_logs-haproxy-dev
Suppress_Type_Name On

[OUTPUT]
Name stdout
Match haproxy.access
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
local hexCharset = "0123456789abcdef"
local function randHex(length)
if length > 0 then
local index = math.random(1, #hexCharset)
return randHex(length - 1) .. hexCharset:sub(index, index)
else
return ""
end
end

local function format_haproxy(c)
if c.error then
return string.format(
"%s [%s] \"%s HTTP/1.1\" %s %s \"%s\"",
c.remote,
os.date("%d/%b/%Y:%H:%M:%S %z"),
c.method,
c.code,
c.size,
c.error
)
else
return string.format(
"%s [%s] \"%s %s HTTP/1.1\" %s %s \"%s\"",
c.remote,
os.date("%d/%b/%Y:%H:%M:%S %z"),
c.method,
c.path,
c.code,
c.size,
c.agent
)
end
end

local formats = {
["haproxy.access"] = format_haproxy

}

function convert_to_otel(tag, timestamp, record)
local data = {
traceId=randHex(32),
spanId=randHex(16),
["@timestamp"]=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
observedTimestamp=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
body=formats[tag](record),
attributes={
data_stream={
dataset=tag,
namespace="production",
type="logs"
}
},
event={
category="web",
name="access",
domain=tag,
kind="event",
result="success",
type="access"
},
http={
request={
method=record.method
},
response={
bytes=tonumber(record.size),
status_code=tonumber(record.code)
},
flavor="1.1",
url=record.path
},
communication={
source={
address=record.remote
}
}
}
if record.remote then
data.communication.source.ip = string.match(record.remote, "%d+%.%d+%.%d+%.%d+")
end
return 1, timestamp, data
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[PARSER]
Name haproxy
Format regex
Regex ^(?<remote>[^ ]*) \[(?<time>[^\]]*)\] (?<frontend>[^ ]*) (?<Backend>[^ ]*) (?<logasap>[^ ]*) (?<code>[^ ]*) (?<size>[^ ]*) (?<reqcookie>[^ ]*) (?<rescookie>[^ ]*) (?<terminationstate>[^ ]*) (?<fc_hex_c_ca_res>[^ ]*) (?<sq_bq>[^ ]*) (((?<agent>[^\"]*) "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?)|(?<error>[^ ]*))?"
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S.%L
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
global
stats socket /var/run/api.sock user haproxy group haproxy mode 660 level admin expose-fd listeners
log stdout format raw local0 info

defaults
mode http
option httplog
timeout client 10s
timeout connect 5s
timeout server 10s
timeout http-request 10s
log global

frontend frontend
bind *:8405
stats enable
stats refresh 5s
capture request header User-Agent len 128
default_backend webservers

backend webservers
server s1 flask-app:5000 check

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Copy access.log, error.log, metrics.log, parsers.conf, and fluent-bit.conf into the /tmp directory
sudo docker run -it -v /tmp/:/tmp/ fluent/fluent-bit /bin/fluent-bit -R /tmp/parsers.conf -c /tmp/fluent-bit.conf
Loading

0 comments on commit 37d574d

Please sign in to comment.