diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index d3a9653..87fc3b7 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -12,7 +12,7 @@ namespace TimberWinR.UnitTests { [TestFixture] public class GrokFilterTests - { + { [Test] public void TestMatch() { @@ -25,24 +25,28 @@ namespace TimberWinR.UnitTests {"ComputerName", "dev.vistaprint.net"} }; - string grokJson = @"{ - ""TimberWinR"":{ - ""Filters"":[ - { - ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", - ""match"":[ - ""Text"", - """" - ], - ""add_field"":[ - ""host"", - ""%{ComputerName}"" - ] - } - }] - } - }"; + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""add_tag"":[ + ""rn_%{Index}"", + ""bar"" + ], + ""add_field"":[ + ""host"", + ""%{ComputerName}"" + ] + } + }] + } + }"; Configuration c = Configuration.FromString(grokJson); @@ -50,7 +54,191 @@ namespace TimberWinR.UnitTests Assert.IsTrue(grok.Apply(json)); + // Verify host field added Assert.AreEqual(json["host"].ToString(), "dev.vistaprint.net"); + + // Verify two tags added + Assert.AreEqual(json["tags"][0].ToString(), "rn_7"); + Assert.AreEqual(json["tags"][1].ToString(), "bar"); + } + + [Test] + public void TestRemoveFields() + { + JObject json = new JObject + { + {"LogFilename", @"C:\\Logs1\\test1.log"}, + {"Index", 7}, + {"Text", null}, + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"} + }; + + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""remove_field"":[ + ""Index"", + ""LogFilename"" + ] + } + }] + } + }"; + + Configuration c = Configuration.FromString(grokJson); + + Grok grok = c.Filters.First() as Grok; + + Assert.IsTrue(grok.Apply(json)); + + // Verify index removed + Assert.IsNull(json["Index"]); + + // Verify index removed + Assert.IsNull(json["LogFilename"]); + } + + [Test] + public void TestConditions() + { + JObject json = new JObject + { + {"LogFilename", @"C:\\Logs1\\test1.log"}, + {"Index", 7}, + {"Text", null}, + {"tags", new JArray + { + "tag1", + "tag2" + } + }, + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"} + }; + + string grokJson1 = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""remove_tag"":[ + ""tag1"" + ] + } + }] + } + }"; + + string grokJson2 = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type].Contains(\""Win32-FileLog\"")"", + ""match"":[ + ""Text"", + """" + ], + ""remove_tag"":[ + ""tag1"" + ] + } + }] + } + }"; + + + string grokJson3 = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type].Contains(\""Win32-Filelog\"")"", + ""match"":[ + ""Text"", + """" + ], + ""remove_tag"":[ + ""tag1"" + ] + } + }] + } + }"; + + // Postitive Tests + Configuration c = Configuration.FromString(grokJson1); + Grok grok = c.Filters.First() as Grok; + Assert.IsTrue(grok.Apply(json)); + + c = Configuration.FromString(grokJson2); + grok = c.Filters.First() as Grok; + Assert.IsTrue(grok.Apply(json)); + + // Negative Test + c = Configuration.FromString(grokJson3); + grok = c.Filters.First() as Grok; + Assert.IsFalse(grok.Apply(json)); + + } + + [Test] + public void TestRemoveTags() + { + JObject json = new JObject + { + {"LogFilename", @"C:\\Logs1\\test1.log"}, + {"Index", 7}, + {"Text", null}, + {"tags", new JArray + { + "tag1", + "tag2" + } + }, + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"} + }; + + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""remove_tag"":[ + ""tag1"" + ] + } + }] + } + }"; + + Configuration c = Configuration.FromString(grokJson); + + Grok grok = c.Filters.First() as Grok; + + Assert.IsTrue(grok.Apply(json)); + + Assert.IsTrue(json["tags"].Children().Count() == 1); + Assert.AreEqual(json["tags"][0].ToString(), "tag2"); } } } diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index 37d0045..128e9aa 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -41,6 +41,8 @@ namespace TimberWinR.Parser { AddFields(json); AddTags(json); + RemoveFields(json); + RemoveTags(json); return true; } return false; @@ -138,6 +140,18 @@ namespace TimberWinR.Parser } } + private void RemoveFields(Newtonsoft.Json.Linq.JObject json) + { + if (RemoveField != null && RemoveField.Length > 0) + { + for (int i = 0; i < RemoveField.Length; i++) + { + string fieldName = ExpandField(RemoveField[i], json); + RemoveProperties(json, new string[] { fieldName }); + } + } + } + private void AddTags(Newtonsoft.Json.Linq.JObject json) { if (AddTag != null && AddTag.Length > 0) @@ -157,5 +171,26 @@ namespace TimberWinR.Parser } } } + + private void RemoveTags(Newtonsoft.Json.Linq.JObject json) + { + if (RemoveTag != null && RemoveTag.Length > 0) + { + JToken tags = json["tags"]; + if (tags != null) + { + List children = tags.Children().ToList(); + for (int i = 0; i < RemoveTag.Length; i++) + { + string tagName = ExpandField(RemoveTag[i], json); + foreach(JToken token in children) + { + if (token.ToString() == tagName) + token.Remove(); + } + } + } + } + } } } diff --git a/TimberWinR/Inputs/TailFileInputListener.cs b/TimberWinR/Inputs/TailFileInputListener.cs index c04e390..6780ef8 100644 --- a/TimberWinR/Inputs/TailFileInputListener.cs +++ b/TimberWinR/Inputs/TailFileInputListener.cs @@ -53,7 +53,7 @@ namespace TimberWinR.Inputs string computerName = System.Environment.MachineName + "." + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") - .GetValue("Domain", ".") + .GetValue("Domain", "") .ToString(); var firstQuery = true; diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 50a8b21..d58a887 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -24,6 +24,29 @@ namespace TimberWinR.Parser } } + protected void RemoveProperties(JToken token, string[] fields) + { + JContainer container = token as JContainer; + if (container == null) return; + + List removeList = new List(); + foreach (JToken el in container.Children()) + { + JProperty p = el as JProperty; + if (p != null && fields.Contains(p.Name)) + { + removeList.Add(el); + } + RemoveProperties(el, fields); + } + + foreach (JToken el in removeList) + { + el.Remove(); + } + } + + protected void ReplaceProperty(JObject json, string propertyName, string propertyValue) { if (json[propertyName] != null) @@ -265,6 +288,9 @@ namespace TimberWinR.Parser [JsonProperty("condition")] public string Condition { get; set; } + [JsonProperty("drop_if_match")] + public bool DropIfMatch { get; set; } + [JsonProperty("match")] public string[] Match { get; set; } @@ -272,7 +298,13 @@ namespace TimberWinR.Parser public string[] AddTag { get; set; } [JsonProperty("add_field")] - public string[] AddField { get; set; } + public string[] AddField { get; set; } + + [JsonProperty("remove_field")] + public string[] RemoveField { get; set; } + + [JsonProperty("remove_tag")] + public string[] RemoveTag { get; set; } } public class Date : LogstashFilter