Added rename to json and grok filters.

Release candidate 1.3.25.0

TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj

@@ -126,6 +126,15 @@
     <Content Include="results5.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="test6-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test6.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results6.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

TimberWinR.TestGenerator/UdpTestGenerator.cs

@@ -59,21 +59,25 @@ namespace TimberWinR.TestGenerator
                     {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
                     {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"},                
                     {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
-                    {"Index", "logstash"}
+                    {"Index", "logstash"},
+                    {"HashedFields", "Application,Executable,RenderedMessage,Team,RecordNumber,Host,Message,Index"}
                 };
 
                 string hashedString = "";
-                foreach(var key in o)
+                if (o["HashedFields"] != null)
                 {
-                    hashedString += key.ToString();
+                    foreach (var key in o["HashedFields"].ToString().Split(new char[] {','}))
+                    {
+                        hashedString += string.Format("{0}:{1}", key, o[key].ToString());
+                    }
+
+                    var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
+                    var md5 = new MD5CryptoServiceProvider().ComputeHash(source);
+                    var hash = string.Concat(md5.Select(x => x.ToString("X2")));
+
+                    o["md5"] = hash;
                 }
 
-                var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
-                var md5 = new MD5CryptoServiceProvider().ComputeHash(source);
-                var hash = string.Concat(md5.Select(x => x.ToString("X2")));
-
-                o["md5"] = hash;
-
                 byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None));
                 IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
                 s.SendTo(sendbuf, ep);

TimberWinR.TestGenerator/results6.json

@@ -0,0 +1,13 @@
+{
+    "Results": {
+        "Inputs": [           
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 80000",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 30"              
+                }
+            }
+        ]
+    }
+}

TimberWinR.TestGenerator/test1-twconfig.json

@@ -38,10 +38,11 @@
                 {
                     "_comment": "Change the host to your Redis instance",
                     "port": 6379,
+                     "max_queue_size":  20000,
                     "batch_count": 500,
                     "threads": 1,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test4-tw.json

@@ -39,9 +39,10 @@
                     "_comment": "Change the host to your Redis instance",
                     "port": 6379,
                     "batch_count": 500,
+                    "max_queue_size":  200,
                     "threads": 2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test4.json

@@ -1,11 +1,14 @@
 {
     "test": "Test 4",
     "arguments": {
+         "--start":  "",
         "--testFile": "test4.json",                
         "--testDir": "test4",
         "--timberWinRConfig": "test4-tw.json",
         "--numMessages": 1234,
-        "--logLevel": "debug",         
+        "--logLevel": "debug",  
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],       
         "--resultsFile": "results4.json"
     }
 }

TimberWinR.TestGenerator/test5.json

@@ -5,7 +5,7 @@
         "--testFile": "test5.json",
         "--testDir": "test5",
         "--timberWinRConfig": "test5-twconfig.json",
-        "--numMessages": 80000,
+        "--numMessages": 1000000,
         "--logLevel": "debug",
         "--udp-host":  "localhost",
         "--udp": "5140",  

TimberWinR.TestGenerator/test6-tw.json

@@ -0,0 +1,51 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Tcp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                },
+                "json": {
+                    "type": "Win32-TailFile",
+                    "source": "Text",
+                    "promote": "Text"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                     "max_queue_size":  20000,
+                    "batch_count": 500,
+                    "threads": 1,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}

TimberWinR.TestGenerator/test6.json

@@ -0,0 +1,15 @@
+{
+    "test": "Test 6",
+    "arguments": {
+        "--start":  "",
+        "--testFile": "test6.json",
+        "--testDir": "test6",
+        "--timberWinRConfig": "test6-tw.json",
+        "--numMessages": 10000,
+        "--logLevel": "debug",
+        "--tcp-host":  "localhost",
+        "--tcp-rate":  1,
+        "--tcp": "5140",               
+        "--resultsFile": "results1.json"
+    }
+}

TimberWinR.UnitTests/GrokFilterTests.cs

@@ -367,6 +367,55 @@ namespace TimberWinR.UnitTests
             Assert.AreEqual(json["tags"][0].ToString(), "tag2");
         }
 
+        [Test]
+        public void TestRename()
+        {
+            JObject json = new JObject
+            {
+                {"LogFilename", @"C:\\Logs1\\test1.log"},
+                {"Index", 7},
+                {"Text", null},
+                {"tags", new JArray
+                    {
+                        "tag1",
+                        "tag2"
+                    }
+                },
+                {"type", "Win32-FileLog"},
+                {"ComputerName", "dev.mycompany.net"}
+            };
+
+            string grokJson = @"{  
+            ""TimberWinR"":{        
+                ""Filters"":[  
+	                {  
+		            ""mutate"":{  
+		                ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
+		                ""match"":[  
+			                ""Text"",
+			                """"
+		                ],
+                        ""rename"":[  
+                            ""type"", ""newtype""
+                        ]		               
+		            }
+	                }]
+                }
+            }";
+
+            Configuration c = Configuration.FromString(grokJson);
+
+            Mutate grok = c.Filters.First() as Mutate;
+
+            Assert.IsTrue(grok.Apply(json));
+
+            Assert.IsNull(json["type"]);
+            Assert.AreEqual("Win32-FileLog", json["newtype"].ToString());
+            Assert.IsTrue(json["tags"].Children().Count() == 2);
+        
+        }
+
+
         [Test]
         public void TestMatchCount()
         {
@@ -476,5 +525,8 @@ namespace TimberWinR.UnitTests
             Assert.IsTrue(grok.AddTag.Length >= 1);
         }
 
+
+       
+
     }
 }

TimberWinR/Filters/GrokFilter.cs

@@ -45,6 +45,7 @@ namespace TimberWinR.Parser
                          new JProperty("condition", Condition),
                          new JProperty("addfields", AddField),
                          new JProperty("addtags", AddTag),
+                         new JProperty("rename", Rename),
                          new JProperty("drop", DropIfMatch),
                          new JProperty("type", Type),
                          new JProperty("removefields", RemoveField),
@@ -85,6 +86,7 @@ namespace TimberWinR.Parser
                 AddTags(json);
                 RemoveFields(json);
                 RemoveTags(json);
+                DoRenames(json);
             }
 
             return true;
@@ -125,6 +127,19 @@ namespace TimberWinR.Parser
             return false; // Not specified is failure
         }
 
+        private void DoRenames(Newtonsoft.Json.Linq.JObject json)
+        {
+            if (Rename != null && Rename.Length > 0)
+            {
+                for (int i = 0; i < Rename.Length; i += 2)
+                {
+                    string oldName = ExpandField(Rename[i], json);
+                    string newName = ExpandField(Rename[i + 1], json);
+                    RenameProperty(json, oldName, newName);
+                }
+            }
+        }
+
         private void AddFields(Newtonsoft.Json.Linq.JObject json)
         {
             if (AddField != null && AddField.Length > 0)

TimberWinR/Filters/JsonFilter.cs

@@ -30,6 +30,7 @@ namespace TimberWinR.Parser
                        new JProperty("promote", Source),
                        new JProperty("target", Target),
                        new JProperty("type", Type),
+                       new JProperty("rename", Rename),
                        new JProperty("addfields", AddField),
                        new JProperty("addtags", AddTag),
                        new JProperty("removefields", RemoveField),

TimberWinR/Parser.cs

@@ -803,6 +803,9 @@ namespace TimberWinR.Parser
         [JsonProperty("remove_tag")]
         public string[] RemoveTag { get; set; }
 
+        [JsonProperty("rename")]
+        public string[] Rename { get; set; }
+
         public override void Validate()
         {
             if (Match == null || Match.Length % 2 != 0)

chocolateyUninstall.ps1.template

@@ -2,6 +2,6 @@ $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in
 $installerType = 'msi' #only one of these: exe, msi, msu
 $scriptPath =  $(Split-Path $MyInvocation.MyCommand.Path)
 $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi'
-$silentArgs = '{CC4DF908-07C4-4BD8-A9FA-6E6AC315E30B} /quiet' 
+$silentArgs = '{267F6E29-2B1B-4BFF-BE88-AC6DE2B50E06} /quiet' 
 $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
 UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes