TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added JSON Filter.

Browse Source
This commit is contained in:
Eric Fontana
2014-09-12 12:22:34 -04:00
parent 60fdea78e3
commit 81d57db90d
11 changed files with 445 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -27,6 +27,7 @@ The current list of supported filters are:
1. [Grok](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md) 1. [Grok](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md)
2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md) 2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md) 3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
4. [Json](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
## JSON ## JSON
Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.2.0.0")] [assembly: AssemblyVersion("1.2.1.0")]
[assembly: AssemblyFileVersion("1.2.0.0")] [assembly: AssemblyFileVersion("1.2.1.0")]

86
TimberWinR.UnitTests/JsonFilterTests.cs Normal file
View File

@@ -0,0 +1,86 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using NUnit.Framework;
using TimberWinR.Parser;
using Newtonsoft.Json.Linq;
namespace TimberWinR.UnitTests
{
[TestFixture]
public class JsonFilterTests
{
[Test]
public void TestDropConditions()
{
JObject jsonInputLine1 = new JObject
{
{"type", "Win32-FileLog"},
{"ComputerName", "dev.vistaprint.net"},
{"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
};
JObject jsonInputLine2 = new JObject
{
{"type", "Win32-FileLog"},
{"ComputerName", "dev.vistaprint.net"},
{"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
};
string jsonFilter = @"{
""TimberWinR"":{
""Filters"":[
{
""json"":{
""type"": ""Win32-FileLog"",
""target"": ""stuff"",
""source"": ""Text""
}
}]
}
}";
string jsonFilterNoTarget = @"{
""TimberWinR"":{
""Filters"":[
{
""json"":{
""type"": ""Win32-FileLog"",
""source"": ""Text""
}
}]
}
}";
// Positive Tests
Configuration c = Configuration.FromString(jsonFilter);
Json jf = c.Filters.First() as Json;
Assert.IsTrue(jf.Apply(jsonInputLine1));
JObject stuff = jsonInputLine1["stuff"] as JObject;
Assert.IsNotNull(stuff);
// 4 fields, Email, Active, CreatedDate, Roles
Assert.AreEqual(4, stuff.Count);
// Now, merge it into the root (starts as 3 fields, ends up with 7 fields)
Assert.AreEqual(3, jsonInputLine2.Count);
c = Configuration.FromString(jsonFilterNoTarget);
jf = c.Filters.First() as Json;
Assert.IsTrue(jf.Apply(jsonInputLine2));
JObject nostuff = jsonInputLine2["stuff"] as JObject;
Assert.IsNull(nostuff);
Assert.AreEqual(7, jsonInputLine2.Count);
var o1 = jsonInputLine1.ToString();
var o2 = jsonInputLine2.ToString();
}
}
}

1
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -52,6 +52,7 @@
<ItemGroup> <ItemGroup>
<Compile Include="Configuration.cs" /> <Compile Include="Configuration.cs" />
<Compile Include="DateFilterTests.cs" /> <Compile Include="DateFilterTests.cs" />
<Compile Include="JsonFilterTests.cs" />
<Compile Include="GrokFilterTests.cs" /> <Compile Include="GrokFilterTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup> </ItemGroup>

9
TimberWinR/Filters/DateFilter.cs
View File

@@ -16,8 +16,15 @@ namespace TimberWinR.Parser
{ {
public override bool Apply(JObject json) public override bool Apply(JObject json)
{ {
if (!string.IsNullOrEmpty(Type))
{
JToken json_type = json["type"];
if (json_type != null && json_type.ToString() != Type)
return true; // Filter does not apply.
}
if (Condition != null && !EvaluateCondition(json, Condition)) if (Condition != null && !EvaluateCondition(json, Condition))
return false; return true;
if (Matches(json)) if (Matches(json))
{ {

134
TimberWinR/Filters/JsonFilter.cs Normal file
View File

@@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace TimberWinR.Parser
{
public partial class Json : LogstashFilter
{
public override bool Apply(JObject json)
{
if (!string.IsNullOrEmpty(Type))
{
JToken json_type = json["type"];
if (json_type != null && json_type.ToString() != Type)
return true; // Filter does not apply.
}
if (Condition != null)
{
var expr = EvaluateCondition(json, Condition);
if (!expr)
return true;
}
var source = json[Source];
if (source != null && !string.IsNullOrEmpty(source.ToString()))
{
try
{
JObject subJson;
if (Target != null && !string.IsNullOrEmpty(Target))
{
subJson = new JObject();
subJson[Target] = JObject.Parse(source.ToString());
}
else
subJson = JObject.Parse(source.ToString());
json.Merge(subJson, new JsonMergeSettings
{
MergeArrayHandling = MergeArrayHandling.Union
});
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
return true;
}
}
AddFields(json);
AddTags(json);
RemoveFields(json);
RemoveTags(json);
return true;
}
private void AddFields(Newtonsoft.Json.Linq.JObject json)
{
if (AddField != null && AddField.Length > 0)
{
for (int i = 0; i < AddField.Length; i += 2)
{
string fieldName = ExpandField(AddField[i], json);
string fieldValue = ExpandField(AddField[i + 1], json);
AddOrModify(json, fieldName, fieldValue);
}
}
}
private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
{
if (RemoveField != null && RemoveField.Length > 0)
{
for (int i = 0; i < RemoveField.Length; i++)
{
string fieldName = ExpandField(RemoveField[i], json);
RemoveProperties(json, new string[] { fieldName });
}
}
}
private void AddTags(Newtonsoft.Json.Linq.JObject json)
{
if (AddTag != null && AddTag.Length > 0)
{
for (int i = 0; i < AddTag.Length; i++)
{
string value = ExpandField(AddTag[i], json);
JToken tags = json["tags"];
if (tags == null)
json.Add("tags", new JArray(value));
else
{
JArray a = tags as JArray;
a.Add(value);
}
}
}
}
private void RemoveTags(Newtonsoft.Json.Linq.JObject json)
{
if (RemoveTag != null && RemoveTag.Length > 0)
{
JToken tags = json["tags"];
if (tags != null)
{
List<JToken> children = tags.Children().ToList();
for (int i = 0; i < RemoveTag.Length; i++)
{
string tagName = ExpandField(RemoveTag[i], json);
foreach (JToken token in children)
{
if (token.ToString() == tagName)
token.Remove();
}
}
}
}
}
}
}

15
TimberWinR/Filters/MutateFilter.cs
View File

@@ -13,8 +13,19 @@ namespace TimberWinR.Parser
{ {
public override bool Apply(JObject json) public override bool Apply(JObject json)
{ {
if (Condition != null && !EvaluateCondition(json, Condition)) if (!string.IsNullOrEmpty(Type))
return false; {
JToken json_type = json["type"];
if (json_type != null && json_type.ToString() != Type)
return true; // Filter does not apply.
}
if (Condition != null)
{
var expr = EvaluateCondition(json, Condition);
if (!expr)
return true;
}
ApplySplits(json); ApplySplits(json);
ApplyRenames(json); ApplyRenames(json);

1
TimberWinR/Inputs/LogsListener.cs
View File

@@ -9,6 +9,7 @@ using Interop.MSUtil;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization; using Newtonsoft.Json.Serialization;
using NLog; using NLog;
using LogQuery = Interop.MSUtil.LogQueryClassClass; using LogQuery = Interop.MSUtil.LogQueryClassClass;

67
TimberWinR/Parser.cs
View File

@@ -561,6 +561,9 @@ namespace TimberWinR.Parser
} }
} }
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("condition")] [JsonProperty("condition")]
public string Condition { get; set; } public string Condition { get; set; }
@@ -597,6 +600,9 @@ namespace TimberWinR.Parser
public partial class Mutate : LogstashFilter public partial class Mutate : LogstashFilter
{ {
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("condition")] [JsonProperty("condition")]
public string Condition { get; set; } public string Condition { get; set; }
@@ -614,7 +620,62 @@ namespace TimberWinR.Parser
} }
} }
public partial class Json : LogstashFilter
{
public class JsonMissingSourceException : Exception
{
public JsonMissingSourceException()
: base("JSON filter source is required")
{
}
}
public class JsonAddFieldException : Exception
{
public JsonAddFieldException()
: base("JSON filter add_field requires tuples")
{
}
}
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("source")]
public string Source { get; set; }
[JsonProperty("target")]
public string Target { get; set; }
[JsonProperty("add_tag")]
public string[] AddTag { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
[JsonProperty("remove_field")]
public string[] RemoveField { get; set; }
[JsonProperty("remove_tag")]
public string[] RemoveTag { get; set; }
public override void Validate()
{
if (string.IsNullOrEmpty(Source))
throw new JsonMissingSourceException();
if (AddField != null && AddField.Length % 2 != 0)
throw new JsonAddFieldException();
}
}
public class Filter public class Filter
{ {
[JsonProperty("grok")] [JsonProperty("grok")]
@@ -625,6 +686,10 @@ namespace TimberWinR.Parser
[JsonProperty("date")] [JsonProperty("date")]
public DateFilter Date { get; set; } public DateFilter Date { get; set; }
[JsonProperty("json")]
public Json Json { get; set; }
} }
public class TimberWinR public class TimberWinR

2
TimberWinR/TimberWinR.csproj
View File

@@ -71,6 +71,7 @@
<Compile Include="Filters\DateFilter.cs" /> <Compile Include="Filters\DateFilter.cs" />
<Compile Include="Filters\FilterBase.cs" /> <Compile Include="Filters\FilterBase.cs" />
<Compile Include="Filters\GrokFilter.cs" /> <Compile Include="Filters\GrokFilter.cs" />
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" /> <Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" /> <Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" /> <Compile Include="Inputs\IISW3CInputListener.cs" />
@@ -105,6 +106,7 @@
</None> </None>
<None Include="mdocs\DateFilter.md" /> <None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" /> <None Include="mdocs\Filters.md" />
<None Include="mdocs\JsonFilter.md" />
<None Include="mdocs\GrokFilter.md" /> <None Include="mdocs\GrokFilter.md" />
<None Include="mdocs\ElasticsearchOutput.md" /> <None Include="mdocs\ElasticsearchOutput.md" />
<None Include="mdocs\RedisOutput.md" /> <None Include="mdocs\RedisOutput.md" />

131
TimberWinR/mdocs/JsonFilter.md Normal file
View File

@@ -0,0 +1,131 @@
# Json Filter
The Json filter allows you to parse a single line of Json into its corresponding fields. This is
particularly useful when parsing log files.
## Json Operations
The following operations are allowed when mutating a field.
| Operation | Type | Description
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *type* | property:string |Type to which this filter applyes, if empty, applies to all types.
| *condition* | property:string |C# expression
| *source* | property:string |Required field indicates which field contains the Json to be parsed
| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
| *add_tag* | property:array |If the filter is successful, add an arbitrary tag to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax.
| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
## Operation Details
### source
The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
argument. In the below example, the message is spected to be something like this from a fictional sample log:
Given this input configuration:
Lets assume that a newline such as the following is appended to foo.jlog:
```
{"Email":"james@example.com","Active":true,"CreatedDate":"2013-01-20T00:00:00Z","Roles":["User","Admin"]}
```
```json
"Inputs": {
"Logs": [
{
"location": "C:\\Logs1\\foo.jlog",
"recurse": -1
}
]
},
"Filters":[
{
"json":{
"type": "Win32-FileLog",
"target": "stuff",
"source": "Text"
}
}]
}
```
In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*
The resulting output would be:
```
{
"type": "Win32-FileLog",
"ComputerName": "dev.vistaprint.net",
"Text": "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}",
"stuff": {
"Email": "james@example.com",
"Active": true,
"CreatedDate": "2013-01-20T00:00:00Z",
"Roles": [
"User",
"Admin"
]
}
}
```
### add_field ["fieldName", "fieldValue", ...]
The fields must be in pairs with fieldName first and value second.
```json
"Filters": [
{
"grok": {
"add_field": [
"ComputerName", "Host",
"Username", "%{SID}"
]
}
}
]
```
### remove_field ["tag1", "tag2", ...]
Remove the fields. More than one field can be specified at a time.
```json
"Filters": [
{
"grok": {
"remove_tag": [
"static_tag1",
"Computer_%{Host}"
]
}
}
]
```
### add_tag ["tag1", "tag2", ...]
Adds the tag(s) to the tag array.
```json
"Filters": [
{
"grok": {
"add_tag": [
"foo_%{Host}",
"static_tag1"
]
}
}
]
```
### remove_tag ["tag1", "tag2", ...]
Remove the tag(s) to the tag array. More than one tag can be specified at a time.
```json
"Filters": [
{
"grok": {
"remove_tag": [
"static_tag1",
"Username"
]
}
}
]
```