From 81d57db90de5f9fd1fba1937e421d3bd159e77aa Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Fri, 12 Sep 2014 12:22:34 -0400 Subject: [PATCH] Added JSON Filter. --- README.md | 1 + .../Properties/AssemblyInfo.cs | 4 +- TimberWinR.UnitTests/JsonFilterTests.cs | 86 +++++++++++ .../TimberWinR.UnitTests.csproj | 1 + TimberWinR/Filters/DateFilter.cs | 9 +- TimberWinR/Filters/JsonFilter.cs | 134 ++++++++++++++++++ TimberWinR/Filters/MutateFilter.cs | 15 +- TimberWinR/Inputs/LogsListener.cs | 1 + TimberWinR/Parser.cs | 67 ++++++++- TimberWinR/TimberWinR.csproj | 2 + TimberWinR/mdocs/JsonFilter.md | 131 +++++++++++++++++ 11 files changed, 445 insertions(+), 6 deletions(-) create mode 100644 TimberWinR.UnitTests/JsonFilterTests.cs create mode 100644 TimberWinR/Filters/JsonFilter.cs create mode 100644 TimberWinR/mdocs/JsonFilter.md diff --git a/README.md b/README.md index 2ac9a8d..2e87af1 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ The current list of supported filters are: 1. [Grok](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md) 2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md) 3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md) + 4. [Json](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md) ## JSON Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 4a6b5cc..cfbb195 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.2.0.0")] -[assembly: AssemblyFileVersion("1.2.0.0")] +[assembly: AssemblyVersion("1.2.1.0")] +[assembly: AssemblyFileVersion("1.2.1.0")] diff --git a/TimberWinR.UnitTests/JsonFilterTests.cs b/TimberWinR.UnitTests/JsonFilterTests.cs new file mode 100644 index 0000000..ec7bbd6 --- /dev/null +++ b/TimberWinR.UnitTests/JsonFilterTests.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using NUnit.Framework; +using TimberWinR.Parser; +using Newtonsoft.Json.Linq; + +namespace TimberWinR.UnitTests +{ + [TestFixture] + public class JsonFilterTests + { + [Test] + public void TestDropConditions() + { + JObject jsonInputLine1 = new JObject + { + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"}, + {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"} + }; + + JObject jsonInputLine2 = new JObject + { + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"}, + {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"} + }; + + + string jsonFilter = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""json"":{ + ""type"": ""Win32-FileLog"", + ""target"": ""stuff"", + ""source"": ""Text"" + } + }] + } + }"; + + string jsonFilterNoTarget = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""json"":{ + ""type"": ""Win32-FileLog"", + ""source"": ""Text"" + } + }] + } + }"; + + // Positive Tests + Configuration c = Configuration.FromString(jsonFilter); + Json jf = c.Filters.First() as Json; + Assert.IsTrue(jf.Apply(jsonInputLine1)); + + JObject stuff = jsonInputLine1["stuff"] as JObject; + Assert.IsNotNull(stuff); + + // 4 fields, Email, Active, CreatedDate, Roles + Assert.AreEqual(4, stuff.Count); + + + + // Now, merge it into the root (starts as 3 fields, ends up with 7 fields) + Assert.AreEqual(3, jsonInputLine2.Count); + c = Configuration.FromString(jsonFilterNoTarget); + jf = c.Filters.First() as Json; + Assert.IsTrue(jf.Apply(jsonInputLine2)); + JObject nostuff = jsonInputLine2["stuff"] as JObject; + Assert.IsNull(nostuff); + Assert.AreEqual(7, jsonInputLine2.Count); + + var o1 = jsonInputLine1.ToString(); + var o2 = jsonInputLine2.ToString(); + } + } +} diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj index 712bdbd..bfae22b 100644 --- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj +++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj @@ -52,6 +52,7 @@ + diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs index d751c05..3c4ba12 100644 --- a/TimberWinR/Filters/DateFilter.cs +++ b/TimberWinR/Filters/DateFilter.cs @@ -16,8 +16,15 @@ namespace TimberWinR.Parser { public override bool Apply(JObject json) { + if (!string.IsNullOrEmpty(Type)) + { + JToken json_type = json["type"]; + if (json_type != null && json_type.ToString() != Type) + return true; // Filter does not apply. + } + if (Condition != null && !EvaluateCondition(json, Condition)) - return false; + return true; if (Matches(json)) { diff --git a/TimberWinR/Filters/JsonFilter.cs b/TimberWinR/Filters/JsonFilter.cs new file mode 100644 index 0000000..0ae6676 --- /dev/null +++ b/TimberWinR/Filters/JsonFilter.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; + +namespace TimberWinR.Parser +{ + public partial class Json : LogstashFilter + { + public override bool Apply(JObject json) + { + if (!string.IsNullOrEmpty(Type)) + { + JToken json_type = json["type"]; + if (json_type != null && json_type.ToString() != Type) + return true; // Filter does not apply. + } + + if (Condition != null) + { + var expr = EvaluateCondition(json, Condition); + if (!expr) + return true; + } + + var source = json[Source]; + + if (source != null && !string.IsNullOrEmpty(source.ToString())) + { + try + { + JObject subJson; + + if (Target != null && !string.IsNullOrEmpty(Target)) + { + subJson = new JObject(); + subJson[Target] = JObject.Parse(source.ToString()); + } + else + subJson = JObject.Parse(source.ToString()); + + json.Merge(subJson, new JsonMergeSettings + { + MergeArrayHandling = MergeArrayHandling.Union + }); + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + return true; + } + } + + AddFields(json); + AddTags(json); + RemoveFields(json); + RemoveTags(json); + + return true; + } + + private void AddFields(Newtonsoft.Json.Linq.JObject json) + { + if (AddField != null && AddField.Length > 0) + { + for (int i = 0; i < AddField.Length; i += 2) + { + string fieldName = ExpandField(AddField[i], json); + string fieldValue = ExpandField(AddField[i + 1], json); + AddOrModify(json, fieldName, fieldValue); + } + } + } + + private void RemoveFields(Newtonsoft.Json.Linq.JObject json) + { + if (RemoveField != null && RemoveField.Length > 0) + { + for (int i = 0; i < RemoveField.Length; i++) + { + string fieldName = ExpandField(RemoveField[i], json); + RemoveProperties(json, new string[] { fieldName }); + } + } + } + + private void AddTags(Newtonsoft.Json.Linq.JObject json) + { + if (AddTag != null && AddTag.Length > 0) + { + for (int i = 0; i < AddTag.Length; i++) + { + string value = ExpandField(AddTag[i], json); + + JToken tags = json["tags"]; + if (tags == null) + json.Add("tags", new JArray(value)); + else + { + JArray a = tags as JArray; + a.Add(value); + } + } + } + } + + private void RemoveTags(Newtonsoft.Json.Linq.JObject json) + { + if (RemoveTag != null && RemoveTag.Length > 0) + { + JToken tags = json["tags"]; + if (tags != null) + { + List children = tags.Children().ToList(); + for (int i = 0; i < RemoveTag.Length; i++) + { + string tagName = ExpandField(RemoveTag[i], json); + foreach (JToken token in children) + { + if (token.ToString() == tagName) + token.Remove(); + } + } + } + } + } + + } +} diff --git a/TimberWinR/Filters/MutateFilter.cs b/TimberWinR/Filters/MutateFilter.cs index 90865b4..755d7f9 100644 --- a/TimberWinR/Filters/MutateFilter.cs +++ b/TimberWinR/Filters/MutateFilter.cs @@ -13,8 +13,19 @@ namespace TimberWinR.Parser { public override bool Apply(JObject json) { - if (Condition != null && !EvaluateCondition(json, Condition)) - return false; + if (!string.IsNullOrEmpty(Type)) + { + JToken json_type = json["type"]; + if (json_type != null && json_type.ToString() != Type) + return true; // Filter does not apply. + } + + if (Condition != null) + { + var expr = EvaluateCondition(json, Condition); + if (!expr) + return true; + } ApplySplits(json); ApplyRenames(json); diff --git a/TimberWinR/Inputs/LogsListener.cs b/TimberWinR/Inputs/LogsListener.cs index 648184f..00a606b 100644 --- a/TimberWinR/Inputs/LogsListener.cs +++ b/TimberWinR/Inputs/LogsListener.cs @@ -9,6 +9,7 @@ using Interop.MSUtil; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; + using NLog; using LogQuery = Interop.MSUtil.LogQueryClassClass; diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index fd0a8ed..6c4c487 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -561,6 +561,9 @@ namespace TimberWinR.Parser } } + [JsonProperty("type")] + public string Type { get; set; } + [JsonProperty("condition")] public string Condition { get; set; } @@ -597,6 +600,9 @@ namespace TimberWinR.Parser public partial class Mutate : LogstashFilter { + [JsonProperty("type")] + public string Type { get; set; } + [JsonProperty("condition")] public string Condition { get; set; } @@ -614,7 +620,62 @@ namespace TimberWinR.Parser } } - + + public partial class Json : LogstashFilter + { + public class JsonMissingSourceException : Exception + { + public JsonMissingSourceException() + : base("JSON filter source is required") + { + } + } + + public class JsonAddFieldException : Exception + { + public JsonAddFieldException() + : base("JSON filter add_field requires tuples") + { + } + } + + + [JsonProperty("type")] + public string Type { get; set; } + + [JsonProperty("condition")] + public string Condition { get; set; } + + [JsonProperty("source")] + public string Source { get; set; } + + [JsonProperty("target")] + public string Target { get; set; } + + [JsonProperty("add_tag")] + public string[] AddTag { get; set; } + + [JsonProperty("add_field")] + public string[] AddField { get; set; } + + [JsonProperty("remove_field")] + public string[] RemoveField { get; set; } + + [JsonProperty("remove_tag")] + public string[] RemoveTag { get; set; } + + public override void Validate() + { + if (string.IsNullOrEmpty(Source)) + throw new JsonMissingSourceException(); + + if (AddField != null && AddField.Length % 2 != 0) + throw new JsonAddFieldException(); + + } + } + + public class Filter { [JsonProperty("grok")] @@ -625,6 +686,10 @@ namespace TimberWinR.Parser [JsonProperty("date")] public DateFilter Date { get; set; } + + [JsonProperty("json")] + public Json Json { get; set; } + } public class TimberWinR diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 38eeda7..b69a6f1 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -71,6 +71,7 @@ + @@ -105,6 +106,7 @@ + diff --git a/TimberWinR/mdocs/JsonFilter.md b/TimberWinR/mdocs/JsonFilter.md new file mode 100644 index 0000000..97c5de9 --- /dev/null +++ b/TimberWinR/mdocs/JsonFilter.md @@ -0,0 +1,131 @@ +# Json Filter +The Json filter allows you to parse a single line of Json into its corresponding fields. This is +particularly useful when parsing log files. + +## Json Operations +The following operations are allowed when mutating a field. + +| Operation | Type | Description +| :---------------|:----------------|:-----------------------------------------------------------------------| +| *type* | property:string |Type to which this filter applyes, if empty, applies to all types. +| *condition* | property:string |C# expression +| *source* | property:string |Required field indicates which field contains the Json to be parsed +| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target* +| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. +| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax. +| *add_tag* | property:array |If the filter is successful, add an arbitrary tag to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax. +| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax. + +## Operation Details +### source +The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second +argument. In the below example, the message is spected to be something like this from a fictional sample log: + +Given this input configuration: + +Lets assume that a newline such as the following is appended to foo.jlog: +``` + {"Email":"james@example.com","Active":true,"CreatedDate":"2013-01-20T00:00:00Z","Roles":["User","Admin"]} +``` + +```json + "Inputs": { + "Logs": [ + { + "location": "C:\\Logs1\\foo.jlog", + "recurse": -1 + } + ] + }, + "Filters":[ + { + "json":{ + "type": "Win32-FileLog", + "target": "stuff", + "source": "Text" + } + }] + } +``` + +In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed +to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff* + +The resulting output would be: +``` + { + "type": "Win32-FileLog", + "ComputerName": "dev.vistaprint.net", + "Text": "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}", + "stuff": { + "Email": "james@example.com", + "Active": true, + "CreatedDate": "2013-01-20T00:00:00Z", + "Roles": [ + "User", + "Admin" + ] + } + } +``` + +### add_field ["fieldName", "fieldValue", ...] +The fields must be in pairs with fieldName first and value second. +```json + "Filters": [ + { + "grok": { + "add_field": [ + "ComputerName", "Host", + "Username", "%{SID}" + ] + } + } + ] +``` + +### remove_field ["tag1", "tag2", ...] +Remove the fields. More than one field can be specified at a time. +```json + "Filters": [ + { + "grok": { + "remove_tag": [ + "static_tag1", + "Computer_%{Host}" + ] + } + } + ] +``` + + +### add_tag ["tag1", "tag2", ...] +Adds the tag(s) to the tag array. +```json + "Filters": [ + { + "grok": { + "add_tag": [ + "foo_%{Host}", + "static_tag1" + ] + } + } + ] +``` + +### remove_tag ["tag1", "tag2", ...] +Remove the tag(s) to the tag array. More than one tag can be specified at a time. +```json + "Filters": [ + { + "grok": { + "remove_tag": [ + "static_tag1", + "Username" + ] + } + } + ] +```