Improve dot notation support

.travis.yml

@@ -1,21 +1,16 @@
 ---
-sudo: false
-language: ruby
-cache: bundler
+sudo: required
+services: docker
+addons:
+  apt:
+    packages:
+    - docker-ce
 matrix:
   include:
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=master
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=7.0
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.7
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.6
-  - rvm: jruby-1.7.27
-    env: LOGSTASH_BRANCH=5.6
+  - env: ELASTIC_STACK_VERSION=7.x
+  - env: ELASTIC_STACK_VERSION=6.x
+  - env: SNAPSHOT=true ELASTIC_STACK_VERSION=7.x
+  - env: SNAPSHOT=true ELASTIC_STACK_VERSION=8.x
   fast_finish: true
-install: true
-script: ci/build.sh
-jdk: openjdk8
-before_install: gem install bundler -v '< 2'
+install: ci/unit/docker-setup.sh
+script: ci/unit/docker-run.sh

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.0.1
+ - Fixed support for deep dot notation [#73](https://github.com/logstash-plugins/logstash-codec-cef/pull/73)
+
 ## 6.0.0
  - Removed obsolete `sev` and `deprecated_v1_fields` fields
 

ci/unit/Dockerfile

@@ -0,0 +1,12 @@
+ARG ELASTIC_STACK_VERSION
+FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION
+COPY --chown=logstash:logstash Gemfile /usr/share/plugins/this/Gemfile
+COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/this/
+RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
+ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin"
+ENV LOGSTASH_SOURCE=1
+ENV JARS_SKIP="true"
+RUN gem install bundler -v '< 2'
+WORKDIR /usr/share/plugins/this
+RUN bundle install
+COPY --chown=logstash:logstash . /usr/share/plugins/this

ci/unit/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+
+# run tests:  cd ci/unit; docker-compose up --build --force-recreate
+# manual:  cd ci/unit; docker-compose run logstash bash
+services:
+
+  logstash:
+    build:
+      context: ../../
+      dockerfile: ci/unit/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+    command: /usr/share/plugins/this/ci/unit/run.sh
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+      LOGSTASH_SOURCE: 1
+      JARS_SKIP: "true"
+      OSS: "true"
+    tty: true
+    #volumes:
+    #  - ./:/usr/share/plugins/this

ci/unit/docker-run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash

ci/unit/docker-setup.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh`
+# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
+set -e
+
+VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json"
+
+if [ "$ELASTIC_STACK_VERSION" ]; then
+    echo "Fetching versions from $VERSION_URL"
+    VERSIONS=$(curl $VERSION_URL)
+    if [[ "$SNAPSHOT" = "true" ]]; then
+      ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
+      echo $ELASTIC_STACK_RETRIEVED_VERSION
+    else
+      ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
+    fi
+    if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
+      # remove starting and trailing double quotes
+      ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
+      ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
+      echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
+      export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
+    fi
+
+    echo "Testing against version: $ELASTIC_STACK_VERSION"
+
+    if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
+        cd /tmp
+
+        if [[ $ELASTIC_STACK_VERSION == 8* ]]; then
+          jq=".build.projects.logstash.packages.\"logstash-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
+        else
+          jq=".build.projects.\"logstash\".packages.\"logstash-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
+        fi
+        echo "curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)"
+        result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)
+        echo $result
+        curl $result > logstash-docker-image.tar.gz
+        tar xfvz logstash-docker-image.tar.gz  repositories
+        echo "Loading docker image: "
+        cat repositories
+        docker load < logstash-docker-image.tar.gz
+        rm logstash-docker-image.tar.gz
+        cd -
+    fi
+
+    if [ -f Gemfile.lock ]; then
+        rm Gemfile.lock
+    fi
+
+    docker-compose -f ci/unit/docker-compose.yml down
+    docker-compose -f ci/unit/docker-compose.yml build
+    #docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash --force-recreate
+else
+    echo "Please set the ELASTIC_STACK_VERSION environment variable"
+    echo "For example: export ELASTIC_STACK_VERSION=6.2.4"
+    exit 1
+fi
+

ci/unit/run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+bundle exec rspec -fd --pattern spec/**/*_spec.rb,spec/**/*_specs.rb

lib/logstash/codecs/cef.rb

@@ -189,13 +189,16 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # commas, periods, and square-bracketed index offsets.
   #
   # To support this, we look for a specific sequence of characters that are followed by an equals sign. This pattern
-  # will correctly identify all strictly-legal keys, and will also match those that include a dot "subkey"
+  # will correctly identify all strictly-legal keys, and will also match those that include a dot-joined "subkeys" and
+  # square-bracketed array indexing
   #
   # That sequence must begin with one or more `\w` (word: alphanumeric + underscore), which _optionally_ may be followed
-  # by "subkey" sequence consisting of a literal dot (`.`) followed by a non-whitespace character, then one or more word
-  # characters, and then one or more characters that do not convey semantic meaning within CEF (e.g., literal-pipe (`|`),
-  # whitespace (`\s`), literal-dot (`.`), literal-equals (`=`), or literal-backslash ('\')).
-  EXTENSION_KEY_PATTERN = /(?:\w+(?:\.[^\s]\w+[^\|\s\.\=\\]+)?(?==))/
+  # by one or more "subkey" sequences and an optional square-bracketed index.
+  #
+  # To be understood by this implementation, a "subkey" sequence must consist of a literal dot (`.`) followed by one or
+  # more characters that do not convey semantic meaning within CEF (e.g., literal-dot (`.`), literal-equals (`=`),
+  # whitespace (`\s`), literal-pipe (`|`), literal-backslash ('\'), or literal-square brackets (`[` or `]`)).
+  EXTENSION_KEY_PATTERN = /(?:\w+(?:\.[^\.=\s\|\\\[\]]+)*(?:\[[0-9]+\])?(?==))/
 
   # Some CEF extension keys seen in the wild use an undocumented array-like syntax that may not be compatible with
   # the Event API's strict-mode FieldReference parser (e.g., `fieldname[0]`).

logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '6.0.0'
+  s.version         = '6.0.1'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the ArcSight Common Event Format (CEF)."

spec/codecs/cef_spec.rb

@@ -555,6 +555,16 @@ def do_decode(codec, data)
       end
     end
 
+    let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
+    it "should be OK with dots in keys" do
+      decode_one(subject, dots_in_keys) do |e|
+        insist { e.get("deviceHostName") } == "loghost"
+        insist { e.get("ad.nn") } == 'TEST'
+        insist { e.get("sourceAddress") } == '192.168.0.1'
+        insist { e.get("destinationPort") } == '53'
+      end
+    end
+
     let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82  spt=1232 dproc=InternetExplorer x.x.x.x'}
     it "should be OK to have one or more spaces in values" do
       decode_one(subject, allow_spaces_in_values) do |e|
@@ -580,6 +590,20 @@ def do_decode(codec, data)
       end
     end
 
+    let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
+    it "should keep ad.fields" do
+      decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("[ad.field][0]") } == "field0"
+        insist { e.get("[ad.foo.name][1]") } == "new_name"
+        insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+        insist { e.get('ad.Error_,Code') } == "3221225578"
+        insist { e.get("additional.dotfieldName") } == "new_value"
+      end
+    end
+
     let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
     it "should correctly parse random values even with additional fields in message" do
       decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|