diff --git a/.nuget/NuGet.Config b/.nuget/NuGet.Config new file mode 100644 index 0000000..67f8ea0 --- /dev/null +++ b/.nuget/NuGet.Config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.nuget/NuGet.targets b/.nuget/NuGet.targets new file mode 100644 index 0000000..3f8c37b --- /dev/null +++ b/.nuget/NuGet.targets @@ -0,0 +1,144 @@ + + + + $(MSBuildProjectDirectory)\..\ + + + false + + + false + + + true + + + false + + + + + + + + + + + $([System.IO.Path]::Combine($(SolutionDir), ".nuget")) + + + + + $(SolutionDir).nuget + + + + $(MSBuildProjectDirectory)\packages.$(MSBuildProjectName.Replace(' ', '_')).config + $(MSBuildProjectDirectory)\packages.$(MSBuildProjectName).config + + + + $(MSBuildProjectDirectory)\packages.config + $(PackagesProjectConfig) + + + + + $(NuGetToolsPath)\NuGet.exe + @(PackageSource) + + "$(NuGetExePath)" + mono --runtime=v4.0.30319 "$(NuGetExePath)" + + $(TargetDir.Trim('\\')) + + -RequireConsent + -NonInteractive + + "$(SolutionDir) " + "$(SolutionDir)" + + + $(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)" $(NonInteractiveSwitch) $(RequireConsentSwitch) -solutionDir $(PaddedSolutionDir) + $(NuGetCommand) pack "$(ProjectPath)" -Properties "Configuration=$(Configuration);Platform=$(Platform)" $(NonInteractiveSwitch) -OutputDirectory "$(PackageOutputDir)" -symbols + + + + RestorePackages; + $(BuildDependsOn); + + + + + $(BuildDependsOn); + BuildPackage; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.nuget/packages.config b/.nuget/packages.config index 7025a72..be55d06 100644 --- a/.nuget/packages.config +++ b/.nuget/packages.config @@ -1,4 +1,7 @@  - + + + + \ No newline at end of file diff --git a/README.md b/README.md index a2979d0..569fd2e 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,6 @@ A single Json filter using the single tag (this is only provided as a convienien } ] ``` - Multiple Json filters must use the jsonFilters and array syntax, also mutateFilters, grokFilters, dateFilters, geoipFilters. ```json "Filters": [ diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs index 73d2b44..94ec34b 100644 --- a/TimberWinR.ServiceHost/Program.cs +++ b/TimberWinR.ServiceHost/Program.cs @@ -7,7 +7,9 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using System.Xml; using Microsoft.Win32; + using TimberWinR.Outputs; using TimberWinR.ServiceHost; using TimberWinR.Inputs; @@ -26,6 +28,9 @@ namespace TimberWinR.ServiceHost private static void Main(string[] args) { + + + Arguments arguments = new Arguments(); HostFactory.Run(hostConfigurator => diff --git a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj index 7ff089c..67f5bf7 100644 --- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj +++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj @@ -12,6 +12,8 @@ v4.0 512 + ..\ + true AnyCPU @@ -49,9 +51,8 @@ - - False - ..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll + + ..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll @@ -87,6 +88,13 @@ + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + \ No newline at end of file diff --git a/TimberWinR.TestGenerator/UdpTestGenerator.cs b/TimberWinR.TestGenerator/UdpTestGenerator.cs new file mode 100644 index 0000000..cf2644f --- /dev/null +++ b/TimberWinR.TestGenerator/UdpTestGenerator.cs @@ -0,0 +1,70 @@ +using System.Threading; +using Newtonsoft.Json.Linq; +using NLog; +using NLog.Config; +using NLog.Targets; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace TimberWinR.TestGenerator +{ + class UdpTestParameters + { + public int Port { get; set; } + public string Host { get; set; } + public int NumMessages { get; set; } + public int SleepTimeMilliseconds { get; set; } + public UdpTestParameters() + { + NumMessages = 100; + Port = 6379; + Host = "localhost"; + SleepTimeMilliseconds = 10; + } + } + + class UdpTestGenerator + { + public static int Generate(UdpTestParameters parms) + { + var hostName = System.Environment.MachineName + "." + + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( + "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString(); + + IPAddress broadcast; + if (!IPAddress.TryParse(parms.Host, out broadcast)) + broadcast = Dns.GetHostEntry(parms.Host).AddressList[0]; + + Socket s = new Socket(broadcast.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + + LogManager.GetCurrentClassLogger().Info("Start UDP Generation"); + + for (int i = 0; i < parms.NumMessages; i++) + { + JObject o = new JObject + { + {"Application", "udp-generator"}, + {"Host", hostName}, + {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, + {"Type", "udp"}, + {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")}, + {"Index", "logstash"} + }; + byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString()); + IPEndPoint ep = new IPEndPoint(broadcast, parms.Port); + s.SendTo(sendbuf, ep); + Thread.Sleep(parms.SleepTimeMilliseconds); + } + + LogManager.GetCurrentClassLogger().Info("Finished UDP Generation"); + + return parms.NumMessages; + } + + } +} diff --git a/TimberWinR.TestGenerator/default.json b/TimberWinR.TestGenerator/default.json new file mode 100644 index 0000000..9740666 --- /dev/null +++ b/TimberWinR.TestGenerator/default.json @@ -0,0 +1,45 @@ +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "_comment": "Output from NLog", + "port": 5140 + } + ], + "TailFiles": [ + { + "interval": 5, + "logSource": "log files", + "location": "*.jlog", + "recurse": -1 + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "\"[EventTypeName]\" == \"Information Event\"", + "match": [ + "Text", + "" + ], + "drop": "true" + } + } + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "batch_count": 500, + "threads": 2, + "host": [ + "tstlexiceapp006.vistaprint.svc" + ] + } + ] + } + } +} diff --git a/TimberWinR.TestGenerator/packages.config b/TimberWinR.TestGenerator/packages.config new file mode 100644 index 0000000..fdebb31 --- /dev/null +++ b/TimberWinR.TestGenerator/packages.config @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/TimberWinR.TestGenerator/results1.json b/TimberWinR.TestGenerator/results1.json new file mode 100644 index 0000000..0e887d1 --- /dev/null +++ b/TimberWinR.TestGenerator/results1.json @@ -0,0 +1,20 @@ +{ + "Results": { + "Inputs": [ + { + "taillog": { + "test1: message sent count": "[messages] == 7404", + "test2: average cpu": "[avgCpuUsage] <= 30", + "test3: maximum memory": "[maxMemUsage] <= 20" + } + }, + { + "udp": { + "test1: message sent count": "[messages] == 1234", + "test2: average cpu": "[avgCpuUsage] <= 30", + "test3: maximum memory": "[maxMemUsage] <= 20" + } + } + ] + } +} diff --git a/TimberWinR.TestGenerator/results2.json b/TimberWinR.TestGenerator/results2.json new file mode 100644 index 0000000..e20ac02 --- /dev/null +++ b/TimberWinR.TestGenerator/results2.json @@ -0,0 +1,20 @@ +{ + "Results": { + "Inputs": [ + { + "taillog": { + "test1: message sent count": "[messages] == 7404", + "test2: average cpu": "[avgCpuUsage] <= 30", + "test3: maximum memory": "[maxMemUsage] <= 15" + } + }, + { + "udp": { + "test1: message sent count": "[messages] == 1234", + "test2: average cpu": "[avgCpuUsage] <= 30", + "test3: maximum memory": "[maxMemUsage] <= 15" + } + } + ] + } +} diff --git a/TimberWinR.TestGenerator/test1-twconfig.json b/TimberWinR.TestGenerator/test1-twconfig.json new file mode 100644 index 0000000..9740666 --- /dev/null +++ b/TimberWinR.TestGenerator/test1-twconfig.json @@ -0,0 +1,45 @@ +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "_comment": "Output from NLog", + "port": 5140 + } + ], + "TailFiles": [ + { + "interval": 5, + "logSource": "log files", + "location": "*.jlog", + "recurse": -1 + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "\"[EventTypeName]\" == \"Information Event\"", + "match": [ + "Text", + "" + ], + "drop": "true" + } + } + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "batch_count": 500, + "threads": 2, + "host": [ + "tstlexiceapp006.vistaprint.svc" + ] + } + ] + } + } +} diff --git a/TimberWinR.TestGenerator/test1.json b/TimberWinR.TestGenerator/test1.json new file mode 100644 index 0000000..241a50c --- /dev/null +++ b/TimberWinR.TestGenerator/test1.json @@ -0,0 +1,15 @@ +{ + "test": "Test 1", + "arguments": { + "--testFile": "test1.json", + "--testDir": "test1", + "--timberWinRConfig": "test1-twconfig.json", + "--numMessages": 1234, + "--logLevel": "debug", + "--udp-host": "::1", + "--udp": "5140", + "--jroll": ["r1.jlog", "r2.jlog"], + "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], + "--resultsFile": "results1.json" + } +} diff --git a/TimberWinR.TestGenerator/test2-tw.json b/TimberWinR.TestGenerator/test2-tw.json new file mode 100644 index 0000000..a321b18 --- /dev/null +++ b/TimberWinR.TestGenerator/test2-tw.json @@ -0,0 +1,45 @@ +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "_comment": "Output from NLog", + "port": 5140 + } + ], + "Logs": [ + { + "interval": 5, + "logSource": "log files", + "location": "*.jlog", + "recurse": -1 + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "\"[EventTypeName]\" == \"Information Event\"", + "match": [ + "Text", + "" + ], + "drop": "true" + } + } + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "batch_count": 500, + "threads": 2, + "host": [ + "tstlexiceapp006.vistaprint.svc" + ] + } + ] + } + } +} diff --git a/TimberWinR.TestGenerator/test2.json b/TimberWinR.TestGenerator/test2.json new file mode 100644 index 0000000..223da98 --- /dev/null +++ b/TimberWinR.TestGenerator/test2.json @@ -0,0 +1,14 @@ +{ + "test": "Test 2", + "arguments": { + "--testFile": "test2.json", + "--testDir": "test2", + "--timberWinRConfig": "test2-tw.json", + "--numMessages": 1234, + "--logLevel": "debug", + "--udp": "5140", + "--jroll": ["r1.jlog", "r2.jlog"], + "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], + "--resultsFile": "results2.json" + } +} diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index c1e4398..5aecd96 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -223,6 +223,7 @@ namespace TimberWinR.UnitTests } }, {"type", "Win32-FileLog"}, + {"Type", "Win32-MyType"}, {"ComputerName", "dev.mycompany.net"} }; @@ -281,11 +282,35 @@ namespace TimberWinR.UnitTests } }"; - // Positive Tests - Configuration c = Configuration.FromString(grokJson1); + string grokJson4 = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""!\""[Type]\"".StartsWith(\""[\"") && !\""[Type]\"".EndsWith(\""]\"") && (\""[type]\"" == \""Win32-FileLog\"")"", + ""match"":[ + ""Text"", + """" + ], + ""remove_tag"":[ + ""tag1"" + ] + } + }] + } + }"; + + + Configuration c = Configuration.FromString(grokJson4); Grok grok = c.Filters.First() as Grok; Assert.IsTrue(grok.Apply(json)); + + // Positive Tests + c = Configuration.FromString(grokJson1); + grok = c.Filters.First() as Grok; + Assert.IsTrue(grok.Apply(json)); + c = Configuration.FromString(grokJson2); grok = c.Filters.First() as Grok; Assert.IsTrue(grok.Apply(json)); diff --git a/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs b/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs deleted file mode 100644 index fa2ff9f..0000000 --- a/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs +++ /dev/null @@ -1,60 +0,0 @@ -namespace TimberWinR.UnitTests.Inputs -{ - using System; - using System.Collections.Generic; - - using Interop.MSUtil; - - using Moq; - - using NUnit.Framework; - - using TimberWinR.Inputs; - using TimberWinR.Parser; - - [TestFixture] - public class IisW3CRowReaderTests : TestBase - { - private IisW3CRowReader reader; - - public override void Setup() - { - base.Setup(); - var fields = new List - { - new Field("date", "DateTime"), - new Field("time", "DateTime"), - new Field("uri") - }; - this.reader = new IisW3CRowReader(fields); - - var recordset = this.GetRecordsetMock(); - this.reader.ReadColumnMap(recordset.Object); - } - - [Test] - public void GivenValidRowAddsTimestampColumn() - { - var record = this.MockRepository.Create(); - record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30)); - record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590)); - record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl"); - - var json = this.reader.ReadToJson(record.Object); - - Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString()); - Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString()); - } - - private Mock GetRecordsetMock() - { - var recordset = this.MockRepository.Create(); - recordset.Setup(x => x.getColumnCount()).Returns(3); - - recordset.Setup(x => x.getColumnName(0)).Returns("date"); - recordset.Setup(x => x.getColumnName(1)).Returns("time"); - recordset.Setup(x => x.getColumnName(2)).Returns("uri"); - return recordset; - } - } -} diff --git a/TimberWinR.UnitTests/TailFileTests.cs b/TimberWinR.UnitTests/TailFileTests.cs index 12a044e..89388a1 100644 --- a/TimberWinR.UnitTests/TailFileTests.cs +++ b/TimberWinR.UnitTests/TailFileTests.cs @@ -28,7 +28,7 @@ namespace TimberWinR.UnitTests var mgr = new Manager(); mgr.LogfileDir = "."; - var tf = new TailFile(); + var tf = new TailFileArguments(); var cancelTokenSource = new CancellationTokenSource(); tf.Location = "TestTailFile1.log"; diff --git a/TimberWinR.UnitTests/TestBase.cs b/TimberWinR.UnitTests/TestBase.cs deleted file mode 100644 index af6612c..0000000 --- a/TimberWinR.UnitTests/TestBase.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace TimberWinR.UnitTests -{ - using Moq; - - using NUnit.Framework; - - public class TestBase - { - public MockRepository MockRepository { get; private set; } - - [SetUp] - public virtual void Setup() - { - this.MockRepository = new MockRepository(MockBehavior.Default); - } - - [TearDown] - public virtual void TearDown() - { - this.MockRepository.VerifyAll(); - } - } -} diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj index b927454..64a2aae 100644 --- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj +++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj @@ -12,6 +12,8 @@ v4.0 512 + ..\ + true true @@ -38,16 +40,13 @@ False ..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll - - ..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll - False - ..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll + ..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll + True - - False - ..\packages\NUnit.2.6.3\lib\nunit.framework.dll + + ..\packages\NUnit.2.6.4\lib\nunit.framework.dll @@ -62,14 +61,12 @@ - - @@ -82,6 +79,7 @@ + Designer @@ -94,7 +92,17 @@ PreserveNewest + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + @@ -103,6 +100,9 @@ $(SolutionDir)\TimberWinR.ExtractID\$(OutDir)\TimberWinR.ExtractID.exe $(TargetDir) $(SolutionDir)chocolateyUninstall.ps1.guid $(SolutionDir)chocolateyUninstall.ps1.template + + cmd.exe /c copy $(SolutionDir)chocolateyUninstall.ps1.template.orig $(SolutionDir)chocolateyUninstall.ps1.template +