From 2e82d8d7b5798ed0bd15da6ac027266ba6991355 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Tue, 16 Sep 2014 07:04:28 -0400 Subject: [PATCH] Fixed bug with multiple filters/inputs being dropped. --- TimberWinR/Configuration.cs | 20 +++++++------- TimberWinR/Diagnostics/Diagnostics.cs | 6 +++- TimberWinR/Filters/DateFilter.cs | 13 ++++++++- TimberWinR/Filters/GrokFilter.cs | 40 +++++++++++++++++++-------- TimberWinR/Filters/JsonFilter.cs | 16 +++++++++++ TimberWinR/Filters/MutateFilter.cs | 13 +++++++++ TimberWinR/Parser.cs | 2 ++ 7 files changed, 86 insertions(+), 24 deletions(-) diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 4497bdd..19afc18 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -127,29 +127,29 @@ namespace TimberWinR if (x.TimberWinR.Inputs != null) { if (x.TimberWinR.Inputs.WindowsEvents != null) - c._events = x.TimberWinR.Inputs.WindowsEvents.ToList(); + c._events.AddRange(x.TimberWinR.Inputs.WindowsEvents.ToList()); if (x.TimberWinR.Inputs.IISW3CLogs != null) - c._iisw3clogs = x.TimberWinR.Inputs.IISW3CLogs.ToList(); + c._iisw3clogs.AddRange(x.TimberWinR.Inputs.IISW3CLogs.ToList()); if (x.TimberWinR.Inputs.Stdins != null) - c._stdins = x.TimberWinR.Inputs.Stdins.ToList(); + c._stdins.AddRange(x.TimberWinR.Inputs.Stdins.ToList()); if (x.TimberWinR.Inputs.Logs != null) - c._logs = x.TimberWinR.Inputs.Logs.ToList(); + c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList()); if (x.TimberWinR.Inputs.Tcps != null) - c._tcps = x.TimberWinR.Inputs.Tcps.ToList(); + c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList()); } if (x.TimberWinR.Outputs != null) { if (x.TimberWinR.Outputs.Redis != null) - c._redisOutputs = x.TimberWinR.Outputs.Redis.ToList(); + c._redisOutputs.AddRange(x.TimberWinR.Outputs.Redis.ToList()); if (x.TimberWinR.Outputs.Elasticsearch != null) - c._elasticsearchOutputs = x.TimberWinR.Outputs.Elasticsearch.ToList(); + c._elasticsearchOutputs.AddRange(x.TimberWinR.Outputs.Elasticsearch.ToList()); if (x.TimberWinR.Outputs.Stdout != null) - c._stdoutOutputs = x.TimberWinR.Outputs.Stdout.ToList(); - } + c._stdoutOutputs.AddRange(x.TimberWinR.Outputs.Stdout.ToList()); + } if (x.TimberWinR.Filters != null) - c._filters = x.TimberWinR.AllFilters.ToList(); + c._filters.AddRange(x.TimberWinR.AllFilters.ToList()); c.Validate(c); diff --git a/TimberWinR/Diagnostics/Diagnostics.cs b/TimberWinR/Diagnostics/Diagnostics.cs index 6b926a7..de14dac 100644 --- a/TimberWinR/Diagnostics/Diagnostics.cs +++ b/TimberWinR/Diagnostics/Diagnostics.cs @@ -92,7 +92,11 @@ namespace TimberWinR.Diagnostics new JProperty("inputs", new JArray( from i in Manager.Listeners - select new JObject(i.ToJson()))), + select new JObject(i.ToJson()))), + new JProperty("filters", + new JArray( + from f in Manager.Config.Filters + select new JObject(f.ToJson()))), new JProperty("outputs", new JArray( from o in Manager.Outputs diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs index 3c4ba12..63a1e61 100644 --- a/TimberWinR/Filters/DateFilter.cs +++ b/TimberWinR/Filters/DateFilter.cs @@ -32,7 +32,18 @@ namespace TimberWinR.Parser } return true; - } + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("date", + new JObject( + new JProperty("condition", Condition), + new JProperty("addfields", AddField) + ))); + return json; + } // copy_field "field1" -> "field2" private void AddFields(Newtonsoft.Json.Linq.JObject json) diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index 7748c9f..65bacb6 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -30,8 +30,24 @@ namespace TimberWinR.Parser } } + + public partial class Grok : LogstashFilter { + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("grok", + new JObject( + new JProperty("condition", Condition), + new JProperty("addfields", AddField), + new JProperty("addtags", AddTag), + new JProperty("removefields", RemoveField), + new JProperty("removetag", RemoveTag) + ))); + return json; + } + // Returns: true - Filter does not apply or has been applied successfully // Returns: false - Drop this object public override bool Apply(JObject json) @@ -42,7 +58,7 @@ namespace TimberWinR.Parser if (json_type != null && json_type.ToString() != Type) return true; // Filter does not apply. } - + if (Matches(json)) { if (Condition != null) @@ -53,22 +69,22 @@ namespace TimberWinR.Parser if (DropIfMatch) return false; // drop this one } - else - return true; + else + return true; } if (DropIfMatch) return false; AddFields(json); - AddTags(json); + AddTags(json); RemoveFields(json); - RemoveTags(json); + RemoveTags(json); } return true; } - + private bool Matches(Newtonsoft.Json.Linq.JObject json) { string field = Match[0]; @@ -110,7 +126,7 @@ namespace TimberWinR.Parser AddOrModify(json, fieldName, fieldValue); } } - } + } private void RemoveFields(Newtonsoft.Json.Linq.JObject json) { @@ -151,17 +167,17 @@ namespace TimberWinR.Parser JToken tags = json["tags"]; if (tags != null) { - List children = tags.Children().ToList(); + List children = tags.Children().ToList(); for (int i = 0; i < RemoveTag.Length; i++) { - string tagName = ExpandField(RemoveTag[i], json); - foreach(JToken token in children) + string tagName = ExpandField(RemoveTag[i], json); + foreach (JToken token in children) { if (token.ToString() == tagName) token.Remove(); } - } - } + } + } } } } diff --git a/TimberWinR/Filters/JsonFilter.cs b/TimberWinR/Filters/JsonFilter.cs index 0ae6676..29b8e22 100644 --- a/TimberWinR/Filters/JsonFilter.cs +++ b/TimberWinR/Filters/JsonFilter.cs @@ -12,6 +12,22 @@ namespace TimberWinR.Parser { public partial class Json : LogstashFilter { + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("json", + new JObject( + new JProperty("condition", Condition), + new JProperty("source", Source), + new JProperty("target", Target), + new JProperty("addfields", AddField), + new JProperty("addtags", AddTag), + new JProperty("removefields", RemoveField), + new JProperty("removetag", RemoveTag) + ))); + return json; + } + public override bool Apply(JObject json) { if (!string.IsNullOrEmpty(Type)) diff --git a/TimberWinR/Filters/MutateFilter.cs b/TimberWinR/Filters/MutateFilter.cs index 755d7f9..6fab38b 100644 --- a/TimberWinR/Filters/MutateFilter.cs +++ b/TimberWinR/Filters/MutateFilter.cs @@ -11,6 +11,19 @@ namespace TimberWinR.Parser { public partial class Mutate : LogstashFilter { + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("mutate", + new JObject( + new JProperty("condition", Condition), + new JProperty("splits", Split), + new JProperty("rename", Rename), + new JProperty("replace", Replace) + ))); + return json; + } + public override bool Apply(JObject json) { if (!string.IsNullOrEmpty(Type)) diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 6c4c487..d66e33e 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -34,6 +34,8 @@ namespace TimberWinR.Parser } } + public abstract JObject ToJson(); + protected bool EvaluateCondition(JObject json, string condition) { // Create a new instance of the C# compiler