diff --git a/README.md b/README.md
index 2ac9a8d..2e87af1 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,7 @@ The current list of supported filters are:
1. [Grok](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md)
2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
+ 4. [Json](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
## JSON
Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be
diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
index 4a6b5cc..cfbb195 100644
--- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
+++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.2.0.0")]
-[assembly: AssemblyFileVersion("1.2.0.0")]
+[assembly: AssemblyVersion("1.2.1.0")]
+[assembly: AssemblyFileVersion("1.2.1.0")]
diff --git a/TimberWinR.UnitTests/JsonFilterTests.cs b/TimberWinR.UnitTests/JsonFilterTests.cs
new file mode 100644
index 0000000..ec7bbd6
--- /dev/null
+++ b/TimberWinR.UnitTests/JsonFilterTests.cs
@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+
+namespace TimberWinR.UnitTests
+{
+ [TestFixture]
+ public class JsonFilterTests
+ {
+ [Test]
+ public void TestDropConditions()
+ {
+ JObject jsonInputLine1 = new JObject
+ {
+ {"type", "Win32-FileLog"},
+ {"ComputerName", "dev.vistaprint.net"},
+ {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
+ };
+
+ JObject jsonInputLine2 = new JObject
+ {
+ {"type", "Win32-FileLog"},
+ {"ComputerName", "dev.vistaprint.net"},
+ {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
+ };
+
+
+ string jsonFilter = @"{
+ ""TimberWinR"":{
+ ""Filters"":[
+ {
+ ""json"":{
+ ""type"": ""Win32-FileLog"",
+ ""target"": ""stuff"",
+ ""source"": ""Text""
+ }
+ }]
+ }
+ }";
+
+ string jsonFilterNoTarget = @"{
+ ""TimberWinR"":{
+ ""Filters"":[
+ {
+ ""json"":{
+ ""type"": ""Win32-FileLog"",
+ ""source"": ""Text""
+ }
+ }]
+ }
+ }";
+
+ // Positive Tests
+ Configuration c = Configuration.FromString(jsonFilter);
+ Json jf = c.Filters.First() as Json;
+ Assert.IsTrue(jf.Apply(jsonInputLine1));
+
+ JObject stuff = jsonInputLine1["stuff"] as JObject;
+ Assert.IsNotNull(stuff);
+
+ // 4 fields, Email, Active, CreatedDate, Roles
+ Assert.AreEqual(4, stuff.Count);
+
+
+
+ // Now, merge it into the root (starts as 3 fields, ends up with 7 fields)
+ Assert.AreEqual(3, jsonInputLine2.Count);
+ c = Configuration.FromString(jsonFilterNoTarget);
+ jf = c.Filters.First() as Json;
+ Assert.IsTrue(jf.Apply(jsonInputLine2));
+ JObject nostuff = jsonInputLine2["stuff"] as JObject;
+ Assert.IsNull(nostuff);
+ Assert.AreEqual(7, jsonInputLine2.Count);
+
+ var o1 = jsonInputLine1.ToString();
+ var o2 = jsonInputLine2.ToString();
+ }
+ }
+}
diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
index 712bdbd..bfae22b 100644
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -52,6 +52,7 @@
+
diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs
index d751c05..3c4ba12 100644
--- a/TimberWinR/Filters/DateFilter.cs
+++ b/TimberWinR/Filters/DateFilter.cs
@@ -16,8 +16,15 @@ namespace TimberWinR.Parser
{
public override bool Apply(JObject json)
{
+ if (!string.IsNullOrEmpty(Type))
+ {
+ JToken json_type = json["type"];
+ if (json_type != null && json_type.ToString() != Type)
+ return true; // Filter does not apply.
+ }
+
if (Condition != null && !EvaluateCondition(json, Condition))
- return false;
+ return true;
if (Matches(json))
{
diff --git a/TimberWinR/Filters/JsonFilter.cs b/TimberWinR/Filters/JsonFilter.cs
new file mode 100644
index 0000000..0ae6676
--- /dev/null
+++ b/TimberWinR/Filters/JsonFilter.cs
@@ -0,0 +1,134 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Xml.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+
+namespace TimberWinR.Parser
+{
+ public partial class Json : LogstashFilter
+ {
+ public override bool Apply(JObject json)
+ {
+ if (!string.IsNullOrEmpty(Type))
+ {
+ JToken json_type = json["type"];
+ if (json_type != null && json_type.ToString() != Type)
+ return true; // Filter does not apply.
+ }
+
+ if (Condition != null)
+ {
+ var expr = EvaluateCondition(json, Condition);
+ if (!expr)
+ return true;
+ }
+
+ var source = json[Source];
+
+ if (source != null && !string.IsNullOrEmpty(source.ToString()))
+ {
+ try
+ {
+ JObject subJson;
+
+ if (Target != null && !string.IsNullOrEmpty(Target))
+ {
+ subJson = new JObject();
+ subJson[Target] = JObject.Parse(source.ToString());
+ }
+ else
+ subJson = JObject.Parse(source.ToString());
+
+ json.Merge(subJson, new JsonMergeSettings
+ {
+ MergeArrayHandling = MergeArrayHandling.Union
+ });
+ }
+ catch (Exception ex)
+ {
+ LogManager.GetCurrentClassLogger().Error(ex);
+ return true;
+ }
+ }
+
+ AddFields(json);
+ AddTags(json);
+ RemoveFields(json);
+ RemoveTags(json);
+
+ return true;
+ }
+
+ private void AddFields(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (AddField != null && AddField.Length > 0)
+ {
+ for (int i = 0; i < AddField.Length; i += 2)
+ {
+ string fieldName = ExpandField(AddField[i], json);
+ string fieldValue = ExpandField(AddField[i + 1], json);
+ AddOrModify(json, fieldName, fieldValue);
+ }
+ }
+ }
+
+ private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (RemoveField != null && RemoveField.Length > 0)
+ {
+ for (int i = 0; i < RemoveField.Length; i++)
+ {
+ string fieldName = ExpandField(RemoveField[i], json);
+ RemoveProperties(json, new string[] { fieldName });
+ }
+ }
+ }
+
+ private void AddTags(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (AddTag != null && AddTag.Length > 0)
+ {
+ for (int i = 0; i < AddTag.Length; i++)
+ {
+ string value = ExpandField(AddTag[i], json);
+
+ JToken tags = json["tags"];
+ if (tags == null)
+ json.Add("tags", new JArray(value));
+ else
+ {
+ JArray a = tags as JArray;
+ a.Add(value);
+ }
+ }
+ }
+ }
+
+ private void RemoveTags(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (RemoveTag != null && RemoveTag.Length > 0)
+ {
+ JToken tags = json["tags"];
+ if (tags != null)
+ {
+ List children = tags.Children().ToList();
+ for (int i = 0; i < RemoveTag.Length; i++)
+ {
+ string tagName = ExpandField(RemoveTag[i], json);
+ foreach (JToken token in children)
+ {
+ if (token.ToString() == tagName)
+ token.Remove();
+ }
+ }
+ }
+ }
+ }
+
+ }
+}
diff --git a/TimberWinR/Filters/MutateFilter.cs b/TimberWinR/Filters/MutateFilter.cs
index 90865b4..755d7f9 100644
--- a/TimberWinR/Filters/MutateFilter.cs
+++ b/TimberWinR/Filters/MutateFilter.cs
@@ -13,8 +13,19 @@ namespace TimberWinR.Parser
{
public override bool Apply(JObject json)
{
- if (Condition != null && !EvaluateCondition(json, Condition))
- return false;
+ if (!string.IsNullOrEmpty(Type))
+ {
+ JToken json_type = json["type"];
+ if (json_type != null && json_type.ToString() != Type)
+ return true; // Filter does not apply.
+ }
+
+ if (Condition != null)
+ {
+ var expr = EvaluateCondition(json, Condition);
+ if (!expr)
+ return true;
+ }
ApplySplits(json);
ApplyRenames(json);
diff --git a/TimberWinR/Inputs/LogsListener.cs b/TimberWinR/Inputs/LogsListener.cs
index 648184f..00a606b 100644
--- a/TimberWinR/Inputs/LogsListener.cs
+++ b/TimberWinR/Inputs/LogsListener.cs
@@ -9,6 +9,7 @@ using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
+
using NLog;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index fd0a8ed..6c4c487 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -561,6 +561,9 @@ namespace TimberWinR.Parser
}
}
+ [JsonProperty("type")]
+ public string Type { get; set; }
+
[JsonProperty("condition")]
public string Condition { get; set; }
@@ -597,6 +600,9 @@ namespace TimberWinR.Parser
public partial class Mutate : LogstashFilter
{
+ [JsonProperty("type")]
+ public string Type { get; set; }
+
[JsonProperty("condition")]
public string Condition { get; set; }
@@ -614,7 +620,62 @@ namespace TimberWinR.Parser
}
}
-
+
+ public partial class Json : LogstashFilter
+ {
+ public class JsonMissingSourceException : Exception
+ {
+ public JsonMissingSourceException()
+ : base("JSON filter source is required")
+ {
+ }
+ }
+
+ public class JsonAddFieldException : Exception
+ {
+ public JsonAddFieldException()
+ : base("JSON filter add_field requires tuples")
+ {
+ }
+ }
+
+
+ [JsonProperty("type")]
+ public string Type { get; set; }
+
+ [JsonProperty("condition")]
+ public string Condition { get; set; }
+
+ [JsonProperty("source")]
+ public string Source { get; set; }
+
+ [JsonProperty("target")]
+ public string Target { get; set; }
+
+ [JsonProperty("add_tag")]
+ public string[] AddTag { get; set; }
+
+ [JsonProperty("add_field")]
+ public string[] AddField { get; set; }
+
+ [JsonProperty("remove_field")]
+ public string[] RemoveField { get; set; }
+
+ [JsonProperty("remove_tag")]
+ public string[] RemoveTag { get; set; }
+
+ public override void Validate()
+ {
+ if (string.IsNullOrEmpty(Source))
+ throw new JsonMissingSourceException();
+
+ if (AddField != null && AddField.Length % 2 != 0)
+ throw new JsonAddFieldException();
+
+ }
+ }
+
+
public class Filter
{
[JsonProperty("grok")]
@@ -625,6 +686,10 @@ namespace TimberWinR.Parser
[JsonProperty("date")]
public DateFilter Date { get; set; }
+
+ [JsonProperty("json")]
+ public Json Json { get; set; }
+
}
public class TimberWinR
diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj
index 38eeda7..b69a6f1 100644
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -71,6 +71,7 @@
+
@@ -105,6 +106,7 @@
+
diff --git a/TimberWinR/mdocs/JsonFilter.md b/TimberWinR/mdocs/JsonFilter.md
new file mode 100644
index 0000000..97c5de9
--- /dev/null
+++ b/TimberWinR/mdocs/JsonFilter.md
@@ -0,0 +1,131 @@
+# Json Filter
+The Json filter allows you to parse a single line of Json into its corresponding fields. This is
+particularly useful when parsing log files.
+
+## Json Operations
+The following operations are allowed when mutating a field.
+
+| Operation | Type | Description
+| :---------------|:----------------|:-----------------------------------------------------------------------|
+| *type* | property:string |Type to which this filter applyes, if empty, applies to all types.
+| *condition* | property:string |C# expression
+| *source* | property:string |Required field indicates which field contains the Json to be parsed
+| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
+| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
+| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
+| *add_tag* | property:array |If the filter is successful, add an arbitrary tag to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax.
+| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
+
+## Operation Details
+### source
+The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
+argument. In the below example, the message is spected to be something like this from a fictional sample log:
+
+Given this input configuration:
+
+Lets assume that a newline such as the following is appended to foo.jlog:
+```
+ {"Email":"james@example.com","Active":true,"CreatedDate":"2013-01-20T00:00:00Z","Roles":["User","Admin"]}
+```
+
+```json
+ "Inputs": {
+ "Logs": [
+ {
+ "location": "C:\\Logs1\\foo.jlog",
+ "recurse": -1
+ }
+ ]
+ },
+ "Filters":[
+ {
+ "json":{
+ "type": "Win32-FileLog",
+ "target": "stuff",
+ "source": "Text"
+ }
+ }]
+ }
+```
+
+In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
+to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*
+
+The resulting output would be:
+```
+ {
+ "type": "Win32-FileLog",
+ "ComputerName": "dev.vistaprint.net",
+ "Text": "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}",
+ "stuff": {
+ "Email": "james@example.com",
+ "Active": true,
+ "CreatedDate": "2013-01-20T00:00:00Z",
+ "Roles": [
+ "User",
+ "Admin"
+ ]
+ }
+ }
+```
+
+### add_field ["fieldName", "fieldValue", ...]
+The fields must be in pairs with fieldName first and value second.
+```json
+ "Filters": [
+ {
+ "grok": {
+ "add_field": [
+ "ComputerName", "Host",
+ "Username", "%{SID}"
+ ]
+ }
+ }
+ ]
+```
+
+### remove_field ["tag1", "tag2", ...]
+Remove the fields. More than one field can be specified at a time.
+```json
+ "Filters": [
+ {
+ "grok": {
+ "remove_tag": [
+ "static_tag1",
+ "Computer_%{Host}"
+ ]
+ }
+ }
+ ]
+```
+
+
+### add_tag ["tag1", "tag2", ...]
+Adds the tag(s) to the tag array.
+```json
+ "Filters": [
+ {
+ "grok": {
+ "add_tag": [
+ "foo_%{Host}",
+ "static_tag1"
+ ]
+ }
+ }
+ ]
+```
+
+### remove_tag ["tag1", "tag2", ...]
+Remove the tag(s) to the tag array. More than one tag can be specified at a time.
+```json
+ "Filters": [
+ {
+ "grok": {
+ "remove_tag": [
+ "static_tag1",
+ "Username"
+ ]
+ }
+ }
+ ]
+```