TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhanced JSON filter and added remove_source and rename properties.

Browse Source
This commit is contained in:
Eric Fontana
2014-09-29 07:38:12 -04:00
parent 19b33a0941
commit 0f660fb937
7 changed files with 74 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@@ -14,6 +14,12 @@ These are broken down into:
2. Filters (Are applied to all Inputs)
3. Outputs (Currently ships only to Redis)
### Support ###
Please use the TimberWinR Google Group for discussion and support:
https://groups.google.com/forum/#!forum/timberwinr
## Input Formats
The current supported Input format sources are:
1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file)
@@ -32,7 +38,7 @@ The current list of supported filters are:
5. [GeoIP](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md)
## JSON
Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be
Since TimberWinR only ships to Redis and Elasticsearch, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be
represented as a JSON Property or Array.
## Supported Output Formats
@@ -86,12 +92,14 @@ This configuration:
2. Filters: Removes the ComputerName field
3. Sends the event to Redis services (server1.host.com, server2.host.com) in a shuffling manner (balanced).
## Installation
You must first install LogParser, then install TimberWinR. Install LogParser from here:
## Installation from Source
If installing from source, you must first install LogParser, then install TimberWinR. Install LogParser from here:
[Install LogParser](http://www.microsoft.com/en-us/download/details.aspx?id=24659) from Microsoft.
After installing, follow the remaining directions here.
After installing, follow the remaining directions here. If you install from [Chocolatey](https://chocolatey.org/packages/TimberWinR) then LogParser will automatically
be installed first.
## Running Interactively
You can run TimberWinR interactively when you are developing your JSON config file, to do so use the
following options:
@@ -99,7 +107,6 @@ following options:
TimberWinR.ServiceHost.exe -configFile:myconfig.json -logLevel:Debug
```
## Automatic Installation via Chocolatey
[TimbeWinR Chocolatey](https://chocolatey.org/packages/TimberWinR)
@@ -191,8 +198,4 @@ sc stop TimberWinR ; stop the service
sc start TimberWinR; start the service
```
### support ###
Please use the TimberWinR Google Group for discussion and support:
https://groups.google.com/forum/#!forum/timberwinr

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.0.0")]
[assembly: AssemblyFileVersion("1.3.0.0")]
[assembly: AssemblyVersion("1.3.1.0")]
[assembly: AssemblyFileVersion("1.3.1.0")]

2
TimberWinR.UnitTests/JsonFilterTests.cs
View File

@@ -75,7 +75,7 @@ namespace TimberWinR.UnitTests
Assert.IsTrue(jf.Apply(jsonInputLine2));
JObject nostuff = jsonInputLine2["stuff"] as JObject;
Assert.IsNull(nostuff);
Assert.AreEqual(7, jsonInputLine2.Count);
Assert.AreEqual(6, jsonInputLine2.Count);
}
}
}

29
TimberWinR/Filters/JsonFilter.cs
View File

@@ -12,6 +12,10 @@ namespace TimberWinR.Parser
{
public partial class Json : LogstashFilter
{
public Json()
{
RemoveSource = true;
}
public override JObject ToJson()
{
JObject json = new JObject(
@@ -45,8 +49,12 @@ namespace TimberWinR.Parser
}
var source = json[Source];
if (source == null)
return true;
if (source != null && !string.IsNullOrEmpty(source.ToString()))
var jsonOrig = source.ToString();
if (!string.IsNullOrEmpty(source.ToString()))
{
try
{
@@ -57,13 +65,26 @@ namespace TimberWinR.Parser
subJson = new JObject();
subJson[Target] = JObject.Parse(source.ToString());
}
else
else
subJson = JObject.Parse(source.ToString());
json.Merge(subJson, new JsonMergeSettings
{
MergeArrayHandling = MergeArrayHandling.Union
MergeArrayHandling = MergeArrayHandling.Replace
});
if (Rename != null && Rename.Length > 0)
{
string oldName = ExpandField(Rename[0], json);
string newName = ExpandField(Rename[1], json);
RenameProperty(json, oldName, newName);
}
if (RemoveSource)
{
RemoveProperties(json, new string[] { Source });
}
}
catch (Exception ex)
{

12
TimberWinR/Parser.cs
View File

@@ -30,7 +30,9 @@ namespace TimberWinR.Parser
if (token != null)
{
json.Remove(oldName);
json.Add(newName, token);
JToken newToken = json[newName];
if (newToken == null)
json.Add(newName, token);
}
}
@@ -741,6 +743,9 @@ namespace TimberWinR.Parser
[JsonProperty("source")]
public string Source { get; set; }
[JsonProperty("remove_source")]
public bool RemoveSource { get; set; }
[JsonProperty("target")]
public string Target { get; set; }
@@ -755,7 +760,10 @@ namespace TimberWinR.Parser
[JsonProperty("remove_tag")]
public string[] RemoveTag { get; set; }
[JsonProperty("rename")]
public string[] Rename { get; set; }
public override void Validate()
{
if (string.IsNullOrEmpty(Source))

1
TimberWinR/mdocs/GrokFilter.md
View File

@@ -28,6 +28,7 @@ The following operations are allowed when mutating a field.
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *type* | property:string |Type to which this filter applyes, if empty, applies to all types.
| *condition* | property:string |C# expression
| *rename* | property:array |Rename one or more fields
| *match* | property:string |Required field must match before any subsequent grok operations are executed.
| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.

25
TimberWinR/mdocs/JsonFilter.md
View File

@@ -9,6 +9,7 @@ The following operations are allowed when mutating a field.
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *type* | property:string |Type to which this filter applies, if empty, applies to all types.
| *condition* | property:string |C# expression, if the expression is true, continue, otherwise, ignore
| *remove_source* | property:bool |If true, the source property is removed, default: true
| *source* | property:string |Required field indicates which field contains the Json to be parsed
| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
@@ -42,7 +43,11 @@ Lets assume that a newline such as the following is appended to foo.jlog:
"json":{
"type": "Win32-FileLog",
"target": "stuff",
"source": "Text"
"source": "Text",
"rename": [
"Text",
"Data"
]
}
}]
}
@@ -68,7 +73,23 @@ The resulting output would be:
}
}
```
### rename ["oldname", "newname", ...]
The fields must be in pairs with oldname first and newname second.
```json
"Filters": [
{
"json":{
"type": "Win32-FileLog",
"target": "stuff",
"source": "Text",
"rename": [
"Text",
"Data"
]
}
}
]
```
### add_field ["fieldName", "fieldValue", ...]
The fields must be in pairs with fieldName first and value second.
```json