Merge pull request #82 from tableau/3.0.2

3.0.2 changes
tableau · Jun 12, 2019 · 68d9531 · 68d9531
2 parents f54a1a5 + 18f044b
commit 68d9531
Show file tree

Hide file tree

Showing 106 changed files with 858 additions and 336 deletions.
diff --git a/ArtifactProcessors/TableauDesktopLogProcessor/Properties/AssemblyInfo.cs b/ArtifactProcessors/TableauDesktopLogProcessor/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Make internal classes visible to the unit test project.
 [assembly: InternalsVisibleTo("Logshark.Tests")]
diff --git a/ArtifactProcessors/TableauServerLogProcessor/Parsers/HttpdParser.cs b/ArtifactProcessors/TableauServerLogProcessor/Parsers/HttpdParser.cs
@@ -25,11 +25,11 @@ public sealed class HttpdParser : AbstractRegexParser, IParser
                             (?<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})\s
                             ""(?<ts_offset>.*?)""\s
                             (?<port>\d{1,5})\s
-                            ""(?<request_method>[A-Z]+)\s(?<resource>.+?)(\sHTTP/(?<http_version>.+?))?""\s
+                            ""((?<request_method>[A-Z]+)\s)?(?<resource>.+?)(\sHTTP\/(?<http_version>.+?))?""\s
                             ""(?<xforwarded_for>.+?)""\s
                             (?<status_code>\d{3})\s
                             (?<response_size>.+?)\s
-                            ""(?<content_length>.*?)""\s
+                            ""((?<content_length>-?[0-9]*?)|(.*?))""\s
                             (?<request_time>\d+)\s
                             (?<request_id>.*)",
                     RegexOptions.ExplicitCapture | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled),
@@ -41,11 +41,11 @@ public sealed class HttpdParser : AbstractRegexParser, IParser
                             (?<ts>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\.\d{3})\s
                             (?<ts_offset>.*?)\s
                             (?<port>\d{1,5})\s
-                            ""(?<request_method>[A-Z]+)\s(?<resource>.+?)(\sHTTP/(?<http_version>.+?))?""\s
+                            ""((?<request_method>[A-Z]+)\s)?(?<resource>.+?)(\sHTTP\/(?<http_version>.+?))?""\s
                             ""(?<xforwarded_for>.+?)""\s
                             (?<status_code>\d{3})\s
                             (?<response_size>.+?)\s
-                            ""(?<content_length>.*?)""\s
+                            ""((?<content_length>-?[0-9]*?)|(.*?))""\s
                             (?<request_time>\d+)\s
                             (?<request_id>.*)",
                     RegexOptions.ExplicitCapture | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled),
@@ -56,11 +56,11 @@ public sealed class HttpdParser : AbstractRegexParser, IParser
                             (?<requester>.+?)\s
                             \[(?<ts>\d+.*?)\s(?<ts_offset>.*?)\]\s
                             (?<port>\d{1,5})\s
-                            ""(?<request_method>[A-Z]+)\s(?<resource>.+?)(\sHTTP/(?<http_version>.+?))?""\s
+                            ""((?<request_method>[A-Z]+)\s)?(?<resource>.+?)(\sHTTP\/(?<http_version>.+?))?""\s
                             ""(?<xforwarded_for>.+?)""\s
                             (?<status_code>\d{3})\s
                             (?<response_size>.+?)\s""
-                            (?<content_length>.*?)""\s
+                            ((?<content_length>-?[0-9]*?)|(.*?))""\s
                             (?<request_time>\d+)\s
                             (?<request_id>.*)",
                     RegexOptions.ExplicitCapture | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled)

diff --git a/ArtifactProcessors/TableauServerLogProcessor/Properties/AssemblyInfo.cs b/ArtifactProcessors/TableauServerLogProcessor/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Make internal classes visible to the unit test project.
 [assembly: InternalsVisibleTo("Logshark.Tests")]
diff --git a/LogParsers.Base/Properties/AssemblyInfo.cs b/LogParsers.Base/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.ArtifactProcessorLib/Properties/AssemblyInfo.cs b/Logshark.ArtifactProcessorLib/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.0.0")]
-[assembly: AssemblyFileVersion("3.0.0.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Required for log4net.
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]

diff --git a/Logshark.ArtifactProcessorModel/Properties/AssemblyInfo.cs b/Logshark.ArtifactProcessorModel/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.CLI.Installer/Product.wxs b/Logshark.CLI.Installer/Product.wxs
@@ -76,6 +76,7 @@
                 <File Id="BouncyCastleCryptoDll" Name="BouncyCastle.Crypto.dll" Vital="yes" Source="$(var.Bin)\BouncyCastle.Crypto.dll" />
                 <File Id="CommandLineDll" Name="CommandLine.dll" Vital="yes" Source="$(var.Bin)\CommandLine.dll" />
                 <File Id="ConcrtDll" Name="concrt140.dll" Vital="yes" Source="$(var.Bin)\concrt140.dll" />
+                <File Id="CrashDumperExe" Name="crashdumper.exe" Vital="yes" Source="$(var.Bin)\crashdumper.exe" />
                 <File Id="CsvHelperDll" Name="CsvHelper.dll" Vital="yes" Source="$(var.Bin)\CsvHelper.dll" />
                 <File Id="DnsClientDll" Name="DnsClient.dll" Vital="yes" Source="$(var.Bin)\DnsClient.dll" />
                 <File Id="HyperdExe" Name="hyperd.exe" Vital="yes" Source="$(var.Bin)\hyperd.exe" />
@@ -117,7 +118,8 @@
                 <File Id="IkvmReflectionDll" Name="IKVM.Reflection.dll" Vital="yes" Source="$(var.Bin)\IKVM.Reflection.dll" />
                 <File Id="IkvmRuntimeDll" Name="IKVM.Runtime.dll" Vital="yes" Source="$(var.Bin)\IKVM.Runtime.dll" />
                 <File Id="IkvmRuntimeJniDll" Name="IKVM.Runtime.JNI.dll" Vital="yes" Source="$(var.Bin)\IKVM.Runtime.JNI.dll" />
-                <File Id="JeMallocDll" Name="jemalloc.dll" Vital="yes" Source="$(var.Bin)\jemalloc.dll" />
+                <File Id="LibCryptoDll" Name="libcrypto-1_1-x64.dll" Vital="yes" Source="$(var.Bin)\libcrypto-1_1-x64.dll" />
+                <File Id="LibSslDll" Name="libssl-1_1-x64.dll" Vital="yes" Source="$(var.Bin)\libssl-1_1-x64.dll" />
                 <File Id="LibPqDll" Name="libpq.dll" Vital="yes" Source="$(var.Bin)\libpq.dll" />
                 <File Id="Log4NetDll" Name="log4net.dll" Vital="yes" Source="$(var.Bin)\log4net.dll" />
                 <File Id="LogParsersBaseDll" Name="LogParsers.Base.dll" Vital="yes" Source="$(var.Bin)\LogParsers.Base.dll" />
@@ -151,8 +153,8 @@
                 <File Id="TableauCommonDll" Name="TableauCommon.dll" Vital="yes" Source="$(var.Bin)\TableauCommon.dll" />
                 <File Id="TableauExtractApiDotNetDll" Name="TableauExtractApiDotNet.dll" Vital="yes" Source="$(var.Bin)\TableauExtractApiDotNet.dll" />
                 <File Id="TableauHyperExtractDll" Name="TableauHyperExtract.dll" Vital="yes" Source="$(var.Bin)\TableauHyperExtract.dll" />
-                <File Id="TabLibEay32Dll" Name="tablibeay32.dll" Vital="yes" Source="$(var.Bin)\tablibeay32.dll" />
-                <File Id="TabSslEay32Dll" Name="tabssleay32.dll" Vital="yes" Source="$(var.Bin)\tabssleay32.dll" />
+                <File Id="TabLibEayDll" Name="tablibeay32.dll" Vital="yes" Source="$(var.Bin)\tablibeay32.dll" />
+                <File Id="TabSslEayDll" Name="tabssleay32.dll" Vital="yes" Source="$(var.Bin)\tabssleay32.dll" />
                 <File Id="TabSysDll" Name="tabsys.dll" Vital="yes" Source="$(var.Bin)\tabsys.dll" />
                 <File Id="TabzDll" Name="tabz.dll" Vital="yes" Source="$(var.Bin)\tabz.dll" />
                 <File Id="TbbDll" Name="tbb.dll" Vital="yes" Source="$(var.Bin)\tbb.dll" />

diff --git a/Logshark.CLI/Config/Logshark-Prod.config b/Logshark.CLI/Config/Logshark-Prod.config
@@ -23,6 +23,7 @@
       <FilePartitioner concurrencyLimit="4" maxFileSizeMb="250" />
       <FileProcessor concurrencyLimitPerCore="1" />
     </Tuning>
+    <!--TempFolder path="C:\Temp\LogShark" /-->
   </RunOptions>
   <ArtifactProcessorOptions>
     <ArtifactProcessor name="DesktopLogProcessor">
@@ -34,6 +35,7 @@
     <ArtifactProcessor name="ServerClassicLogProcessor">
       <DefaultPlugins>
         <Plugin name="Apache" />
+        <Plugin name="Art" />
         <Plugin name="Backgrounder" />
         <Plugin name="ClusterController" />
         <Plugin name="Config" />
@@ -54,6 +56,7 @@
     <ArtifactProcessor name="ServerTsmLogProcessor">
       <DefaultPlugins>
         <Plugin name="Apache" />
+        <Plugin name="Art" />
         <Plugin name="Backgrounder" />
         <Plugin name="ClusterController" />
         <Plugin name="Config" />

diff --git a/Logshark.CLI/Config/Logshark.config b/Logshark.CLI/Config/Logshark.config
@@ -36,6 +36,7 @@
     <ArtifactProcessor name="ServerClassicLogProcessor">
       <DefaultPlugins>
         <Plugin name="Apache" />
+        <Plugin name="Art" />
         <Plugin name="Backgrounder" />
         <Plugin name="ClusterController" />
         <Plugin name="Config" />
@@ -56,6 +57,7 @@
     <ArtifactProcessor name="ServerTsmLogProcessor">
       <DefaultPlugins>
         <Plugin name="Apache" />
+        <Plugin name="Art" />
         <Plugin name="Backgrounder" />
         <Plugin name="ClusterController" />
         <Plugin name="Config" />

diff --git a/Logshark.CLI/Properties/AssemblyInfo.cs b/Logshark.CLI/Properties/AssemblyInfo.cs
@@ -31,8 +31,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Required for log4net.
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]
diff --git a/Logshark.Common/Properties/AssemblyInfo.cs b/Logshark.Common/Properties/AssemblyInfo.cs
@@ -31,5 +31,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.ConfigSection/Properties/AssemblyInfo.cs b/Logshark.ConfigSection/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Make internal classes visible to the unit test project.
 [assembly: InternalsVisibleTo("Logshark.Tests")]
diff --git a/Logshark.ConfigSectionGenerator/Properties/AssemblyInfo.cs b/Logshark.ConfigSectionGenerator/Properties/AssemblyInfo.cs
@@ -31,5 +31,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.ConnectionModel/Properties/AssemblyInfo.cs b/Logshark.ConnectionModel/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Required for log4net.
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]

diff --git a/Logshark.Core/Properties/AssemblyInfo.cs b/Logshark.Core/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Required for log4net.
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]

diff --git a/Logshark.PluginLib/Properties/AssemblyInfo.cs b/Logshark.PluginLib/Properties/AssemblyInfo.cs
@@ -31,5 +31,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.PluginModel/Properties/AssemblyInfo.cs b/Logshark.PluginModel/Properties/AssemblyInfo.cs
@@ -31,5 +31,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
diff --git a/Logshark.RequestModel/Properties/AssemblyInfo.cs b/Logshark.RequestModel/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Make internal classes visible to the unit test project.
 [assembly: InternalsVisibleTo("Logshark.Tests")]
diff --git a/Logshark.Tests/Properties/AssemblyInfo.cs b/Logshark.Tests/Properties/AssemblyInfo.cs
@@ -32,8 +32,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("3.0.1.0")]
-[assembly: AssemblyFileVersion("3.0.1.0")]
+[assembly: AssemblyVersion("3.0.2.0")]
+[assembly: AssemblyFileVersion("3.0.2.0")]
 
 // Required for log4net.
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]
diff --git a/Logshark.Tests/ServerLogProcessorTests/HttpdParserTests.cs b/Logshark.Tests/ServerLogProcessorTests/HttpdParserTests.cs
@@ -47,6 +47,31 @@ public void ParseHttpdLogLine_v8()
             Assert.AreEqual(expectedResult, actualResult);
         }
 
+        [Test, Description("Parse missing request method and bad content length.")]
+        public void ParseHttpdLogLineWithMissingRequestMethodAndBadContentLength()
+        {
+            const string sampleLogLinev2018 =
+                @"10.210.24.3 127.0.0.1 - 2018-05-09T16:07:58.120 ""GMT Daylight Time"" 80 ""/vizql/w/Superstore/v/Overview/bootstrapSession/sessions/185CCDC854A44765BB0298E93B403879-0:3 HTTP/1.1"" ""-"" 200 136026 ""<script>alert(Content-Length)</script>"" 2370951 WvMOzgKIhfzh9kFWO@ow2gAAA1Y";
+            const string expectedResultv2018 =
+                @"{""request_ip"":""10.210.24.3"",""requester"":""127.0.0.1"",""ts"":""2018-05-09T09:07:58.12-07:00"",""ts_offset"":""0100"",""port"":""80"",""resource"":""/vizql/w/Superstore/v/Overview/bootstrapSession/sessions/185CCDC854A44765BB0298E93B403879-0:3"",""http_version"":""1.1"",""status_code"":""200"",""response_size"":""136026"",""request_time"":""2370951"",""request_id"":""WvMOzgKIhfzh9kFWO@ow2gAAA1Y"",""line"":1}";
+            var result = ParserTestHelpers.ParseSingleLine(sampleLogLinev2018, new HttpdParser());
+            Assert.AreEqual(expectedResultv2018, result);
+
+            const string sampleLogLinev9 =
+                @"172.16.115.248 - - 2019-01-10 20:07:14.256 Eastern Standard Time 443 ""BASELINE-CONTROL /jqZPZpAJ.htm HTTP/1.1"" ""45.33.86.32"" 405 246 ""<script>alert(Content-Length)</script>"" 15632 XDfsQrLI3QXJ7VR7OZto1QAAA@M";
+            const string expectedResultv9 =
+                @"{""request_ip"":""172.16.115.248"",""ts"":""2019-01-10T12:07:14.256-08:00"",""ts_offset"":""-0500"",""port"":""443"",""resource"":""BASELINE-CONTROL /jqZPZpAJ.htm"",""http_version"":""1.1"",""xforwarded_for"":""45.33.86.32"",""status_code"":""405"",""response_size"":""246"",""request_time"":""15632"",""request_id"":""XDfsQrLI3QXJ7VR7OZto1QAAA@M"",""line"":1}";
+            result = ParserTestHelpers.ParseSingleLine(sampleLogLinev9, new HttpdParser());
+            Assert.AreEqual(expectedResultv9, result);
+
+            const string sampleLogLinev8 =
+                @"10.17.136.120 - - [25/Feb/2014:02:20:12 -0800] 80 ""/tableau_prefix_local/workbooks/dsvc_test_workbook_8_1_0.twb?language=en HTTP/1.1"" ""10.32.149.80, 10.17.136.120"" 200 1192683 ""<script>alert(Content-Length)</script>"" 218401 UwxuXAoRhHQAACGoBVwAAAGK";
+            const string expectedResultv8 =
+                @"{""request_ip"":""10.17.136.120"",""ts"":""2014-02-24T18:20:12-08:00"",""ts_offset"":""-0800"",""port"":""80"",""resource"":""/tableau_prefix_local/workbooks/dsvc_test_workbook_8_1_0.twb?language=en"",""http_version"":""1.1"",""xforwarded_for"":""10.32.149.80, 10.17.136.120"",""status_code"":""200"",""response_size"":""1192683"",""request_time"":""218401"",""request_id"":""UwxuXAoRhHQAACGoBVwAAAGK"",""line"":1}";
+            result = ParserTestHelpers.ParseSingleLine(sampleLogLinev8, new HttpdParser());
+            Assert.AreEqual(expectedResultv8, result);
+        }
+
         [Test, Description("Parses a sample httpd access logfile and ensures that all documents were parsed correctly.")]
         [TestCase("httpd_access_9x.txt", Description = "access log, 9.x+ Format")]
         public void ParseHttpdAccessFullFile(string logFile)