From 0f660fb93763a8e1c59c01d1ee43b7e815d0de21 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Mon, 29 Sep 2014 07:38:12 -0400 Subject: [PATCH] Enhanced JSON filter and added remove_source and rename properties. --- README.md | 21 ++++++++------ .../Properties/AssemblyInfo.cs | 4 +-- TimberWinR.UnitTests/JsonFilterTests.cs | 2 +- TimberWinR/Filters/JsonFilter.cs | 29 ++++++++++++++++--- TimberWinR/Parser.cs | 12 ++++++-- TimberWinR/mdocs/GrokFilter.md | 1 + TimberWinR/mdocs/JsonFilter.md | 25 ++++++++++++++-- 7 files changed, 74 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6cacf72..2fa3906 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ These are broken down into: 2. Filters (Are applied to all Inputs) 3. Outputs (Currently ships only to Redis) +### Support ### +Please use the TimberWinR Google Group for discussion and support: + +https://groups.google.com/forum/#!forum/timberwinr + + ## Input Formats The current supported Input format sources are: 1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file) @@ -32,7 +38,7 @@ The current list of supported filters are: 5. [GeoIP](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md) ## JSON -Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be +Since TimberWinR only ships to Redis and Elasticsearch, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be represented as a JSON Property or Array. ## Supported Output Formats @@ -86,12 +92,14 @@ This configuration: 2. Filters: Removes the ComputerName field 3. Sends the event to Redis services (server1.host.com, server2.host.com) in a shuffling manner (balanced). -## Installation -You must first install LogParser, then install TimberWinR. Install LogParser from here: +## Installation from Source +If installing from source, you must first install LogParser, then install TimberWinR. Install LogParser from here: [Install LogParser](http://www.microsoft.com/en-us/download/details.aspx?id=24659) from Microsoft. -After installing, follow the remaining directions here. +After installing, follow the remaining directions here. If you install from [Chocolatey](https://chocolatey.org/packages/TimberWinR) then LogParser will automatically +be installed first. + ## Running Interactively You can run TimberWinR interactively when you are developing your JSON config file, to do so use the following options: @@ -99,7 +107,6 @@ following options: TimberWinR.ServiceHost.exe -configFile:myconfig.json -logLevel:Debug ``` - ## Automatic Installation via Chocolatey [TimbeWinR Chocolatey](https://chocolatey.org/packages/TimberWinR) @@ -191,8 +198,4 @@ sc stop TimberWinR ; stop the service sc start TimberWinR; start the service ``` -### support ### -Please use the TimberWinR Google Group for discussion and support: - -https://groups.google.com/forum/#!forum/timberwinr diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 28b8434..83cf33a 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.0.0")] -[assembly: AssemblyFileVersion("1.3.0.0")] +[assembly: AssemblyVersion("1.3.1.0")] +[assembly: AssemblyFileVersion("1.3.1.0")] diff --git a/TimberWinR.UnitTests/JsonFilterTests.cs b/TimberWinR.UnitTests/JsonFilterTests.cs index 2521182..c730b53 100644 --- a/TimberWinR.UnitTests/JsonFilterTests.cs +++ b/TimberWinR.UnitTests/JsonFilterTests.cs @@ -75,7 +75,7 @@ namespace TimberWinR.UnitTests Assert.IsTrue(jf.Apply(jsonInputLine2)); JObject nostuff = jsonInputLine2["stuff"] as JObject; Assert.IsNull(nostuff); - Assert.AreEqual(7, jsonInputLine2.Count); + Assert.AreEqual(6, jsonInputLine2.Count); } } } diff --git a/TimberWinR/Filters/JsonFilter.cs b/TimberWinR/Filters/JsonFilter.cs index 29b8e22..ca052fc 100644 --- a/TimberWinR/Filters/JsonFilter.cs +++ b/TimberWinR/Filters/JsonFilter.cs @@ -12,6 +12,10 @@ namespace TimberWinR.Parser { public partial class Json : LogstashFilter { + public Json() + { + RemoveSource = true; + } public override JObject ToJson() { JObject json = new JObject( @@ -45,8 +49,12 @@ namespace TimberWinR.Parser } var source = json[Source]; + if (source == null) + return true; - if (source != null && !string.IsNullOrEmpty(source.ToString())) + var jsonOrig = source.ToString(); + + if (!string.IsNullOrEmpty(source.ToString())) { try { @@ -57,13 +65,26 @@ namespace TimberWinR.Parser subJson = new JObject(); subJson[Target] = JObject.Parse(source.ToString()); } - else + else subJson = JObject.Parse(source.ToString()); - + json.Merge(subJson, new JsonMergeSettings { - MergeArrayHandling = MergeArrayHandling.Union + MergeArrayHandling = MergeArrayHandling.Replace }); + + if (Rename != null && Rename.Length > 0) + { + string oldName = ExpandField(Rename[0], json); + string newName = ExpandField(Rename[1], json); + RenameProperty(json, oldName, newName); + } + + if (RemoveSource) + { + RemoveProperties(json, new string[] { Source }); + } + } catch (Exception ex) { diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 88a5cc6..2ce4239 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -30,7 +30,9 @@ namespace TimberWinR.Parser if (token != null) { json.Remove(oldName); - json.Add(newName, token); + JToken newToken = json[newName]; + if (newToken == null) + json.Add(newName, token); } } @@ -741,6 +743,9 @@ namespace TimberWinR.Parser [JsonProperty("source")] public string Source { get; set; } + [JsonProperty("remove_source")] + public bool RemoveSource { get; set; } + [JsonProperty("target")] public string Target { get; set; } @@ -755,7 +760,10 @@ namespace TimberWinR.Parser [JsonProperty("remove_tag")] public string[] RemoveTag { get; set; } - + + [JsonProperty("rename")] + public string[] Rename { get; set; } + public override void Validate() { if (string.IsNullOrEmpty(Source)) diff --git a/TimberWinR/mdocs/GrokFilter.md b/TimberWinR/mdocs/GrokFilter.md index e215e23..6bfaa63 100644 --- a/TimberWinR/mdocs/GrokFilter.md +++ b/TimberWinR/mdocs/GrokFilter.md @@ -28,6 +28,7 @@ The following operations are allowed when mutating a field. | :---------------|:----------------|:-----------------------------------------------------------------------| | *type* | property:string |Type to which this filter applyes, if empty, applies to all types. | *condition* | property:string |C# expression +| *rename* | property:array |Rename one or more fields | *match* | property:string |Required field must match before any subsequent grok operations are executed. | *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. | *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax. diff --git a/TimberWinR/mdocs/JsonFilter.md b/TimberWinR/mdocs/JsonFilter.md index 13ce92a..a582046 100644 --- a/TimberWinR/mdocs/JsonFilter.md +++ b/TimberWinR/mdocs/JsonFilter.md @@ -9,6 +9,7 @@ The following operations are allowed when mutating a field. | :---------------|:----------------|:-----------------------------------------------------------------------| | *type* | property:string |Type to which this filter applies, if empty, applies to all types. | *condition* | property:string |C# expression, if the expression is true, continue, otherwise, ignore +| *remove_source* | property:bool |If true, the source property is removed, default: true | *source* | property:string |Required field indicates which field contains the Json to be parsed | *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target* | *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. @@ -42,7 +43,11 @@ Lets assume that a newline such as the following is appended to foo.jlog: "json":{ "type": "Win32-FileLog", "target": "stuff", - "source": "Text" + "source": "Text", + "rename": [ + "Text", + "Data" + ] } }] } @@ -68,7 +73,23 @@ The resulting output would be: } } ``` - +### rename ["oldname", "newname", ...] +The fields must be in pairs with oldname first and newname second. +```json + "Filters": [ + { + "json":{ + "type": "Win32-FileLog", + "target": "stuff", + "source": "Text", + "rename": [ + "Text", + "Data" + ] + } + } + ] +``` ### add_field ["fieldName", "fieldValue", ...] The fields must be in pairs with fieldName first and value second. ```json