From 7156d9c6f878f214bb2f3a6df98ed956cd71063d Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Wed, 13 May 2015 08:03:19 -0400 Subject: [PATCH] Added rename to json and grok filters. --- .../TimberWinR.TestGenerator.csproj | 9 ++++ TimberWinR.TestGenerator/UdpTestGenerator.cs | 22 ++++---- TimberWinR.TestGenerator/results6.json | 13 +++++ TimberWinR.TestGenerator/test1-twconfig.json | 3 +- TimberWinR.TestGenerator/test4-tw.json | 3 +- TimberWinR.TestGenerator/test4.json | 5 +- TimberWinR.TestGenerator/test5.json | 2 +- TimberWinR.TestGenerator/test6-tw.json | 51 ++++++++++++++++++ TimberWinR.TestGenerator/test6.json | 15 ++++++ TimberWinR.UnitTests/GrokFilterTests.cs | 52 +++++++++++++++++++ TimberWinR/Filters/GrokFilter.cs | 15 ++++++ TimberWinR/Filters/JsonFilter.cs | 1 + TimberWinR/Parser.cs | 3 ++ chocolateyUninstall.ps1.template | 2 +- 14 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 TimberWinR.TestGenerator/results6.json create mode 100644 TimberWinR.TestGenerator/test6-tw.json create mode 100644 TimberWinR.TestGenerator/test6.json diff --git a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj index 19d78bb..b394ff4 100644 --- a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj +++ b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj @@ -126,6 +126,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + diff --git a/TimberWinR.TestGenerator/UdpTestGenerator.cs b/TimberWinR.TestGenerator/UdpTestGenerator.cs index 6e93cbf..575897f 100644 --- a/TimberWinR.TestGenerator/UdpTestGenerator.cs +++ b/TimberWinR.TestGenerator/UdpTestGenerator.cs @@ -59,21 +59,25 @@ namespace TimberWinR.TestGenerator {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"}, {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")}, - {"Index", "logstash"} + {"Index", "logstash"}, + {"HashedFields", "Application,Executable,RenderedMessage,Team,RecordNumber,Host,Message,Index"} }; string hashedString = ""; - foreach(var key in o) + if (o["HashedFields"] != null) { - hashedString += key.ToString(); + foreach (var key in o["HashedFields"].ToString().Split(new char[] {','})) + { + hashedString += string.Format("{0}:{1}", key, o[key].ToString()); + } + + var source = ASCIIEncoding.ASCII.GetBytes(hashedString); + var md5 = new MD5CryptoServiceProvider().ComputeHash(source); + var hash = string.Concat(md5.Select(x => x.ToString("X2"))); + + o["md5"] = hash; } - var source = ASCIIEncoding.ASCII.GetBytes(hashedString); - var md5 = new MD5CryptoServiceProvider().ComputeHash(source); - var hash = string.Concat(md5.Select(x => x.ToString("X2"))); - - o["md5"] = hash; - byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None)); IPEndPoint ep = new IPEndPoint(broadcast, parms.Port); s.SendTo(sendbuf, ep); diff --git a/TimberWinR.TestGenerator/results6.json b/TimberWinR.TestGenerator/results6.json new file mode 100644 index 0000000..112222e --- /dev/null +++ b/TimberWinR.TestGenerator/results6.json @@ -0,0 +1,13 @@ +{ + "Results": { + "Inputs": [ + { + "udp": { + "test1: message sent count": "[messages] == 80000", + "test2: average cpu": "[avgCpuUsage] <= 30", + "test3: maximum memory": "[maxMemUsage] <= 30" + } + } + ] + } +} diff --git a/TimberWinR.TestGenerator/test1-twconfig.json b/TimberWinR.TestGenerator/test1-twconfig.json index f6f81d0..76402da 100644 --- a/TimberWinR.TestGenerator/test1-twconfig.json +++ b/TimberWinR.TestGenerator/test1-twconfig.json @@ -38,10 +38,11 @@ { "_comment": "Change the host to your Redis instance", "port": 6379, + "max_queue_size": 20000, "batch_count": 500, "threads": 1, "host": [ - "tstlexiceapp006.mycompany.svc" + "tstlexiceapp006.vistaprint.svc" ] } ] diff --git a/TimberWinR.TestGenerator/test4-tw.json b/TimberWinR.TestGenerator/test4-tw.json index 2a4c5c7..65bf4dd 100644 --- a/TimberWinR.TestGenerator/test4-tw.json +++ b/TimberWinR.TestGenerator/test4-tw.json @@ -39,9 +39,10 @@ "_comment": "Change the host to your Redis instance", "port": 6379, "batch_count": 500, + "max_queue_size": 200, "threads": 2, "host": [ - "tstlexiceapp006.mycompany.svc" + "tstlexiceapp006.vistaprint.svc" ] } ] diff --git a/TimberWinR.TestGenerator/test4.json b/TimberWinR.TestGenerator/test4.json index a4a943a..76d0d21 100644 --- a/TimberWinR.TestGenerator/test4.json +++ b/TimberWinR.TestGenerator/test4.json @@ -1,11 +1,14 @@ { "test": "Test 4", "arguments": { + "--start": "", "--testFile": "test4.json", "--testDir": "test4", "--timberWinRConfig": "test4-tw.json", "--numMessages": 1234, - "--logLevel": "debug", + "--logLevel": "debug", + "--jroll": ["r1.jlog", "r2.jlog"], + "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], "--resultsFile": "results4.json" } } diff --git a/TimberWinR.TestGenerator/test5.json b/TimberWinR.TestGenerator/test5.json index 303f7f0..ccd1ef6 100644 --- a/TimberWinR.TestGenerator/test5.json +++ b/TimberWinR.TestGenerator/test5.json @@ -5,7 +5,7 @@ "--testFile": "test5.json", "--testDir": "test5", "--timberWinRConfig": "test5-twconfig.json", - "--numMessages": 80000, + "--numMessages": 1000000, "--logLevel": "debug", "--udp-host": "localhost", "--udp": "5140", diff --git a/TimberWinR.TestGenerator/test6-tw.json b/TimberWinR.TestGenerator/test6-tw.json new file mode 100644 index 0000000..7781f80 --- /dev/null +++ b/TimberWinR.TestGenerator/test6-tw.json @@ -0,0 +1,51 @@ +{ + "TimberWinR": { + "Inputs": { + "Tcp": [ + { + "_comment": "Output from NLog", + "port": 5140 + } + ], + "TailFiles": [ + { + "interval": 5, + "logSource": "log files", + "location": "*.jlog", + "recurse": -1 + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "\"[EventTypeName]\" == \"Information Event\"", + "match": [ + "Text", + "" + ], + "drop": "true" + }, + "json": { + "type": "Win32-TailFile", + "source": "Text", + "promote": "Text" + } + } + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "max_queue_size": 20000, + "batch_count": 500, + "threads": 1, + "host": [ + "tstlexiceapp006.vistaprint.svc" + ] + } + ] + } + } +} diff --git a/TimberWinR.TestGenerator/test6.json b/TimberWinR.TestGenerator/test6.json new file mode 100644 index 0000000..71dca66 --- /dev/null +++ b/TimberWinR.TestGenerator/test6.json @@ -0,0 +1,15 @@ +{ + "test": "Test 6", + "arguments": { + "--start": "", + "--testFile": "test6.json", + "--testDir": "test6", + "--timberWinRConfig": "test6-tw.json", + "--numMessages": 10000, + "--logLevel": "debug", + "--tcp-host": "localhost", + "--tcp-rate": 1, + "--tcp": "5140", + "--resultsFile": "results1.json" + } +} diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index 5aecd96..bb83c84 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -367,6 +367,55 @@ namespace TimberWinR.UnitTests Assert.AreEqual(json["tags"][0].ToString(), "tag2"); } + [Test] + public void TestRename() + { + JObject json = new JObject + { + {"LogFilename", @"C:\\Logs1\\test1.log"}, + {"Index", 7}, + {"Text", null}, + {"tags", new JArray + { + "tag1", + "tag2" + } + }, + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.mycompany.net"} + }; + + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""mutate"":{ + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""rename"":[ + ""type"", ""newtype"" + ] + } + }] + } + }"; + + Configuration c = Configuration.FromString(grokJson); + + Mutate grok = c.Filters.First() as Mutate; + + Assert.IsTrue(grok.Apply(json)); + + Assert.IsNull(json["type"]); + Assert.AreEqual("Win32-FileLog", json["newtype"].ToString()); + Assert.IsTrue(json["tags"].Children().Count() == 2); + + } + + [Test] public void TestMatchCount() { @@ -476,5 +525,8 @@ namespace TimberWinR.UnitTests Assert.IsTrue(grok.AddTag.Length >= 1); } + + + } } diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index bf334e0..a7310d9 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -45,6 +45,7 @@ namespace TimberWinR.Parser new JProperty("condition", Condition), new JProperty("addfields", AddField), new JProperty("addtags", AddTag), + new JProperty("rename", Rename), new JProperty("drop", DropIfMatch), new JProperty("type", Type), new JProperty("removefields", RemoveField), @@ -85,6 +86,7 @@ namespace TimberWinR.Parser AddTags(json); RemoveFields(json); RemoveTags(json); + DoRenames(json); } return true; @@ -125,6 +127,19 @@ namespace TimberWinR.Parser return false; // Not specified is failure } + private void DoRenames(Newtonsoft.Json.Linq.JObject json) + { + if (Rename != null && Rename.Length > 0) + { + for (int i = 0; i < Rename.Length; i += 2) + { + string oldName = ExpandField(Rename[i], json); + string newName = ExpandField(Rename[i + 1], json); + RenameProperty(json, oldName, newName); + } + } + } + private void AddFields(Newtonsoft.Json.Linq.JObject json) { if (AddField != null && AddField.Length > 0) diff --git a/TimberWinR/Filters/JsonFilter.cs b/TimberWinR/Filters/JsonFilter.cs index fbe3b63..d7bd872 100644 --- a/TimberWinR/Filters/JsonFilter.cs +++ b/TimberWinR/Filters/JsonFilter.cs @@ -30,6 +30,7 @@ namespace TimberWinR.Parser new JProperty("promote", Source), new JProperty("target", Target), new JProperty("type", Type), + new JProperty("rename", Rename), new JProperty("addfields", AddField), new JProperty("addtags", AddTag), new JProperty("removefields", RemoveField), diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 5bdc6e5..7c6257d 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -803,6 +803,9 @@ namespace TimberWinR.Parser [JsonProperty("remove_tag")] public string[] RemoveTag { get; set; } + [JsonProperty("rename")] + public string[] Rename { get; set; } + public override void Validate() { if (Match == null || Match.Length % 2 != 0) diff --git a/chocolateyUninstall.ps1.template b/chocolateyUninstall.ps1.template index 1827afd..28cf2dd 100644 --- a/chocolateyUninstall.ps1.template +++ b/chocolateyUninstall.ps1.template @@ -2,6 +2,6 @@ $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in $installerType = 'msi' #only one of these: exe, msi, msu $scriptPath = $(Split-Path $MyInvocation.MyCommand.Path) $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi' -$silentArgs = '{CC4DF908-07C4-4BD8-A9FA-6E6AC315E30B} /quiet' +$silentArgs = '{267F6E29-2B1B-4BFF-BE88-AC6DE2B50E06} /quiet' $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes