opensearch-project · Swiddis · Dec 4, 2023 · Nov 29, 2023
@@ -0,0 +1 @@
+[{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T06:46:14.029Z","observedTimestamp":"2023-11-20T06:46:14.029Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:06:46:14 +0000] \"GET /monitor HTTP/1.1\" 200 23741 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"300","bytes":23741},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:24.734Z","observedTimestamp":"2023-11-20T07:10:24.734Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:24 +0000] \"GET /monitor HTTP/1.1\" 200 23881 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"400","bytes":23881},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:19.628Z","observedTimestamp":"2023-11-20T07:10:19.628Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:19 +0000] \"GET /monitor HTTP/1.1\" 200 23881 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23881},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:14.505Z","observedTimestamp":"2023-11-20T07:10:14.505Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:14 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:09.302Z","observedTimestamp":"2023-11-20T07:10:09.302Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:09 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:04.151Z","observedTimestamp":null,"body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:04 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"200","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-20T07:10:04.151Z","observedTimestamp":"2023-11-20T07:10:04.000Z","body":"10.81.1.1:57147 stats stats/<STATS>0/0/0/0/0 1/1/0/0/00/0--LR--[20/Nov/2023:07:10:04 +0000] \"GET /monitor HTTP/1.1\" 200 23886 \"{Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0}\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{"method":"GET"},"response":{"status_code":"404","bytes":23886},"url":"/monitor"}},{"span_id":"abcdef1010","attributes":{"data_stream":{"namespace":"production","dataset":"haproxy.access","type":"logs"}},"event":{"domain":"haproxy.access","kind":"event","result":"success","type":["access"],"category":["web"],"name":"access"},"@timestamp":"2023-11-21T08:57:33.000Z","observedTimestamp":"2023-11-21T08:57:43.000Z","body":"10.81.1.1:61108 stats stats/<NOSRV>-1/-1/-1/-1/10009a 1/1/0/0/00/0--cR--[21/Nov/2023:08:57:43 +0000] \" HTTP/1.1\" 408 0 \"\"<BADREQ>\"","trace_id":"102981ABCD2901","http":{"flavor":"1.1","request":{},"response":{"status_code":"408","bytes":1},"url":"/monitor"}}]
@@ -0,0 +1,50 @@
+{
+    "name": "haproxy",
+    "version": "1.0.0",
+    "displayName": "Haproxy Dashboard",
+    "description": "Haproxy logs collector",
+    "license": "Apache-2.0",
+    "type": "logs",
+    "labels": ["Observability", "Logs"],
+    "author": "OpenSearch",
+    "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info",
+    "statics": {
+        "logo": {
+          "annotation": "Haproxy Logo",
+          "path": "logo.svg"
+        },
+        "gallery": [
+          {
+            "annotation": "Haproxy Dashboard",
+            "path": "dashboard1.png"
+          },
+          {
+            "annotation": "Haproxy Dashboard view",
+            "path": "dashboard2.png"
+          }
+        ]
+      },
+    "components": [
+      {
+        "name": "communication",
+        "version": "1.0.0"
+      },
+      {
+        "name": "http",
+        "version": "1.0.0"
+      },
+      {
+        "name": "logs",
+        "version": "1.0.0"
+      }
+    ],
+    "assets": {
+      "savedObjects": {
+        "name": "haproxy",
+        "version": "1.0.0"
+      }
+    },
+    "sampleData": {
+      "path": "sample.json"
+    }
+  }
@@ -0,0 +1,18 @@
+# HAProxy Integration
+
+## What is HAProxy?
+
+HAProxy is open-source software that provides a high availability load balancer and proxy server for TCP and HTTP-based applications.
+
+See additional details [here](http://www.haproxy.org/).
+
+## What is HAProxy Integration?
+
+An integration is a bundle of pre-canned assets that are packaged together in a meaningful manner.
+HAProxy integration includes dashboards, visualisations, queries and an index mapping.
+
+### Dashboards
+
+![](../static/dashboard1.png)
+![](../static/dashboard2.png)
+
@@ -0,0 +1,90 @@
+version: '3'
+
+services:
+  opensearch:
+    image: opensearchstaging/opensearch:3.0.0
+    container_name: opensearch
+    environment:
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=opensearch # Name the node that will run in this container
+      - discovery.seed_hosts=opensearch # Nodes to look for when discovering the cluster
+      - cluster.initial_cluster_manager_nodes=opensearch # Nodes eligibile to serve as cluster manager
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
+      - "DISABLE_INSTALL_DEMO_CONFIG=true" # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
+      - "DISABLE_SECURITY_PLUGIN=true" # Disables security plugin
+    ulimits:
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit)
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
+        hard: 65536
+    volumes:
+      - opensearch:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
+    ports:
+      - 9200:9200
+      - 9600:9600
+    expose:
+      - "9200"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"]
+      interval: 5s
+      timeout: 25s
+      retries: 4
+    networks:
+      - opensearch-net # All of the containers will join the same Docker bridge network
+  haproxy:
+    container_name: haproxy
+    image: haproxytech/haproxy-alpine:2.4
+    volumes:
+      # - ./haproxy-otel/opentelemetry_module.conf:/etc/nginx/conf.d/opentelemetry_module.conf
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    ports: 
+      - 8405:8405
+    depends_on:
+      - flask-app
+      - fluentbit
+    networks:
+      - opensearch-net # All of the containers will join the same Docker bridge network
+    links:
+      - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: 127.0.0.1:24224
+        tag: haproxy.access
+  fluentbit:
+    container_name: fluentbit
+    image: fluent/fluent-bit:latest
+    volumes:
+      - ./fluent-bit:/fluent-bit/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    depends_on:
+      - opensearch
+    networks:
+      - opensearch-net
+  redis:
+    image: redis
+    ports:
+      - 6357:6357
+    networks:
+      - opensearch-net
+  flask-app:
+    build: flask-app
+    ports:
+      - 5000:5000
+    depends_on:
+      - redis
+    volumes:
+      - ./flask-app/app.py:/code/app.py
+    networks:
+      - opensearch-net
+
+volumes:
+  opensearch:
+
+networks:
+  opensearch-net:
@@ -0,0 +1,9 @@
+FROM python:latest
+ADD . /code
+WORKDIR /code
+RUN pip install -r requirements.txt
+
+RUN mkdir -p /run/haproxy/
+RUN export VALUE=$(id -u haproxy)
+
+CMD ["flask", "run", "-h", "0.0.0.0", "-p", "5000", "--debug"]
@@ -0,0 +1,20 @@
+from random import random, randint
+from flask import Flask
+from redis import Redis
+
+app = Flask(__name__)
+redis = Redis(host='redis', port=6379)
+
+
+@app.route("/")
+def hits():
+    if random() < 0.20:
+        return "Random error", 500
+    h = redis.incr('hits')
+    return f"This page has been viewed {h} time(s)"
+
+@app.route("/dice")
+def roll_dice():
+    if random() < 0.05:
+        return "Random error", 500
+    return str(randint(1, 6))
@@ -0,0 +1,2 @@
+flask==2.3.2
+redis
@@ -0,0 +1,30 @@
+[SERVICE]
+    Parsers_File parsers.conf
+
+[INPUT]
+    Name forward
+    Port 24224
+
+[FILTER]
+    Name parser
+    Match haproxy.access
+    Key_Name log
+    Parser haproxy
+
+[FILTER]
+    Name    lua
+    Match   haproxy.access
+    Script  otel-converter.lua
+    call    convert_to_otel
+
+[OUTPUT]
+    Name  opensearch
+    Match haproxy.access
+    Host  opensearch
+    Port  9200
+    Index  ss4o_logs-haproxy-dev
+    Suppress_Type_Name On
+
+[OUTPUT]
+    Name stdout
+    Match haproxy.access
@@ -0,0 +1,84 @@
+local hexCharset = "0123456789abcdef"
+local function randHex(length)
+    if length > 0 then
+        local index = math.random(1, #hexCharset)
+        return randHex(length - 1) .. hexCharset:sub(index, index)
+    else
+        return ""
+    end
+end
+
+local function format_haproxy(c)
+    if c.error then
+        return string.format(
+        "%s [%s] \"%s HTTP/1.1\" %s %s \"%s\"",
+        c.remote,
+        os.date("%d/%b/%Y:%H:%M:%S %z"),
+        c.method,
+        c.code,
+        c.size,
+        c.error
+        )
+    else
+        return string.format(
+        "%s [%s] \"%s %s HTTP/1.1\" %s %s \"%s\"",
+        c.remote,
+        os.date("%d/%b/%Y:%H:%M:%S %z"),
+        c.method,
+        c.path,
+        c.code,
+        c.size,
+        c.agent
+        )
+    end
+end
+
+local formats = {
+    ["haproxy.access"] = format_haproxy
+
+}
+
+function convert_to_otel(tag, timestamp, record)
+    local data = {
+        traceId=randHex(32),
+        spanId=randHex(16),
+        ["@timestamp"]=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
+        observedTimestamp=os.date("!%Y-%m-%dT%H:%M:%S.000Z"),
+        body=formats[tag](record),
+        attributes={
+            data_stream={
+                dataset=tag,
+                namespace="production",
+                type="logs"
+            }
+        },
+        event={
+            category="web",
+            name="access",
+            domain=tag,
+            kind="event",
+            result="success",
+            type="access"
+        },
+        http={
+            request={
+                method=record.method
+            },
+            response={
+                bytes=tonumber(record.size),
+                status_code=tonumber(record.code)
+            },
+            flavor="1.1",
+            url=record.path
+        },
+        communication={
+            source={
+                address=record.remote
+            }
+        }
+    }
+    if record.remote then
+        data.communication.source.ip = string.match(record.remote, "%d+%.%d+%.%d+%.%d+")
+    end
+    return 1, timestamp, data
+end
@@ -0,0 +1,6 @@
+[PARSER]
+    Name   haproxy
+    Format regex
+    Regex ^(?<remote>[^ ]*) \[(?<time>[^\]]*)\] (?<frontend>[^ ]*) (?<Backend>[^ ]*) (?<logasap>[^ ]*) (?<code>[^ ]*) (?<size>[^ ]*) (?<reqcookie>[^ ]*) (?<rescookie>[^ ]*) (?<terminationstate>[^ ]*) (?<fc_hex_c_ca_res>[^ ]*) (?<sq_bq>[^ ]*) (((?<agent>[^\"]*) "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?)|(?<error>[^ ]*))?"
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S.%L
@@ -0,0 +1,23 @@
+global
+  stats socket /var/run/api.sock user haproxy group haproxy mode 660 level admin expose-fd listeners
+  log stdout format raw local0 info
+
+defaults
+  mode http
+  option httplog
+  timeout client 10s
+  timeout connect 5s
+  timeout server 10s
+  timeout http-request 10s
+  log global
+
+frontend frontend
+  bind *:8405
+  stats enable
+  stats refresh 5s
+  capture request header User-Agent len 128
+  default_backend webservers
+
+backend webservers
+  server s1 flask-app:5000 check
+
@@ -0,0 +1,2 @@
+# Copy access.log, error.log, metrics.log, parsers.conf, and fluent-bit.conf into the /tmp directory
+sudo docker run -it -v /tmp/:/tmp/ fluent/fluent-bit /bin/fluent-bit -R /tmp/parsers.conf -c /tmp/fluent-bit.conf