enhancement(regex_parser transform): Add RegexSet support to regex (v…

…ectordotdev#2493) Signed-off-by: Brian Menges <[email protected]>
jacobbraaten · Dec 9, 2020 · c635db1 · c635db1
1 parent d8f7de2
commit c635db1
Show file tree

Hide file tree

Showing 8 changed files with 161 additions and 55 deletions.
diff --git a/.meta/transforms/regex_parser.toml.erb b/.meta/transforms/regex_parser.toml.erb
@@ -38,17 +38,17 @@ If `target_field` is set and the log contains a field of the same name \
 as the target, it will only be overwritten if this is set to `true`.\
 """
 
-[transforms.regex_parser.options.regex]
+[transforms.regex_parser.options.patterns]
 type = "string"
 common = true
 examples = [
 """\
-^(?P<timestamp>[\\w\\-:\\+]+) (?P<level>\\w+) (?P<message>.*)$\
+['^(?P<timestamp>[\\w\\-:\\+]+) (?P<level>\\w+) (?P<message>.*)$']\
 """
 ]
 required = true
 description = """\
-The Regular Expression to apply. Do not include the leading or trailing `/`.\
+The Regular Expressions to apply. Do not include the leading or trailing `/` in any of the expressions.\
 """
 
 [transforms.regex_parser.options.target_field]
@@ -85,7 +85,7 @@ And the following configuration:
 [transforms.<transform-id>]
   type = "regex_parser"
   field = "message"
-  regex = '^(?P<host>[\w\.]+) - (?P<user>[\w]+) (?P<bytes_in>[\d]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$'
+  patterns = ['^(?P<host>[\w\.]+) - (?P<user>[\w]+) (?P<bytes_in>[\d]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$']
 
 [transforms.<transform-id>.types]
   bytes_in = "int"

diff --git a/benches/bench.rs b/benches/bench.rs
@@ -346,7 +346,7 @@ fn benchmark_transforms(c: &mut Criterion) {
                         "parser",
                         &["in"],
                         transforms::regex_parser::RegexParserConfig {
-                            regex: r"status=(?P<status>\d+)".to_string(),
+                            patterns: vec![r"status=(?P<status>\d+)".to_string()],
                             field: None,
                             ..Default::default()
                         },
@@ -410,7 +410,7 @@ fn benchmark_regex(c: &mut Criterion) {
                     let rt = vector::runtime::Runtime::single_threaded().unwrap();
                     let parser =transforms::regex_parser::RegexParserConfig {
                         // Many captures to stress the regex parser
-                        regex: r#"^(?P<addr>\d+\.\d+\.\d+\.\d+) (?P<user>\S+) (?P<auth>\S+) \[(?P<date>\d+/[A-Za-z]+/\d+:\d+:\d+:\d+ [+-]\d{4})\] "(?P<method>[A-Z]+) (?P<uri>[^"]+) HTTP/\d\.\d" (?P<code>\d+) (?P<size>\d+) "(?P<referrer>[^"]+)" "(?P<browser>[^"]+)""#.into(),
+                        patterns: vec![r#"^(?P<addr>\d+\.\d+\.\d+\.\d+) (?P<user>\S+) (?P<auth>\S+) \[(?P<date>\d+/[A-Za-z]+/\d+:\d+:\d+:\d+ [+-]\d{4})\] "(?P<method>[A-Z]+) (?P<uri>[^"]+) HTTP/\d\.\d" (?P<code>\d+) (?P<size>\d+) "(?P<referrer>[^"]+)" "(?P<browser>[^"]+)""#.into()],
                         field: None,
                         drop_failed: true,
                         ..Default::default()
@@ -465,7 +465,7 @@ fn benchmark_complex(c: &mut Criterion) {
                         "parser",
                         &["in1", "in2"],
                         transforms::regex_parser::RegexParserConfig {
-                            regex: r"status=(?P<status>\d+)".to_string(),
+                            patterns: vec![r"status=(?P<status>\d+)".to_string()],
                             field: None,
                             ..Default::default()
                         },

diff --git a/config/examples/docs_example.toml b/config/examples/docs_example.toml
@@ -11,7 +11,7 @@ data_dir = "/var/lib/vector"
 [transforms.apache_parser]
   inputs       = ["apache_logs"]
   type         = "regex_parser"                # fast/powerful regex
-  regex        = '^(?P<host>[w.]+) - (?P<user>[w]+) (?P<bytes_in>[d]+) [(?P<timestamp>.*)] "(?P<method>[w]+) (?P<path>.*)" (?P<status>[d]+) (?P<bytes_out>[d]+)$'
+  patterns      = ['^(?P<host>[w.]+) - (?P<user>[w]+) (?P<bytes_in>[d]+) [(?P<timestamp>.*)] "(?P<method>[w]+) (?P<path>.*)" (?P<status>[d]+) (?P<bytes_out>[d]+)$']
 
 # Sample the data to save on cost
 [transforms.apache_sampler]

diff --git a/config/examples/file_to_cloudwatch_metrics.toml b/config/examples/file_to_cloudwatch_metrics.toml
@@ -14,7 +14,7 @@ start_at_beginning = true
 [transforms.regex_parser]
 inputs = ["file"]
 type = "regex_parser"
-regex = '^(?P<host>[\w\.]+) - (?P<user>[\w-]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$'
+patterns = ['^(?P<host>[\w\.]+) - (?P<user>[\w-]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$']
 
 # Transform into metrics
 [transforms.log_to_metric]

diff --git a/config/examples/file_to_prometheus.toml b/config/examples/file_to_prometheus.toml
@@ -14,7 +14,7 @@ start_at_beginning = true
 [transforms.regex_parser]
 inputs = ["file"]
 type = "regex_parser"
-regex = '^(?P<host>[\w\.]+) - (?P<user>[\w-]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$'
+patterns = ['^(?P<host>[\w\.]+) - (?P<user>[\w-]+) \[(?P<timestamp>.*)\] "(?P<method>[\w]+) (?P<path>.*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+)$']
 
 # Transform into metrics
 [transforms.log_to_metric]