TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-factored to use FilterBase

Browse Source
This commit is contained in:
Eric Fontana
2014-07-22 11:11:40 -04:00
parent dc08f8c6bb
commit 45cda907f7
6 changed files with 64 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/sampleconf.xml
View File

@@ -17,10 +17,10 @@
<!--Single Tag-->
<Match field="Text" value="%{SYSLOGLINE}" />
</Grok>
<Date field="timestamp" target="@timestamp" convertToUTC="true">
<!--<Date field="timestamp" target="@timestamp" convertToUTC="true">
<Pattern>MMM d HH:mm:ss</Pattern>
<Pattern>MMM dd HH:mm:ss</Pattern>
<Pattern>ISO8601</Pattern>
</Date>
</Date>-->
</Filters>
</TimberWinR>

36
TimberWinR.UnitTests/Configuration.cs
View File

@@ -6,6 +6,7 @@ using System.Text;
using System.Threading.Tasks;
using TimberWinR;
using TimberWinR.Inputs;
using TimberWinR.Filters;
namespace TimberWinR.UnitTests
{
@@ -48,10 +49,11 @@ namespace TimberWinR.UnitTests
public void OutputGroks()
{
foreach (var grok in c.Groks.ToArray())
{
Console.WriteLine(grok);
}
//IEnumerable<FilterBase> filters = c.Filters;
//foreach (var grok in c.Filters)
// Console.WriteLine(grok);
}
[Test]
@@ -82,13 +84,7 @@ namespace TimberWinR.UnitTests
public void NumOfIISW3C()
{
Assert.AreEqual(1, c.IISW3C.ToArray().Length);
}
[Test]
public void NumOfGroks()
{
Assert.AreEqual(1, c.Groks.ToArray().Length);
}
}
[Test]
public void FieldsOfEvents()
@@ -322,22 +318,6 @@ namespace TimberWinR.UnitTests
Assert.AreEqual(dirTime, iisw3c.DirTime);
Assert.AreEqual(consolidateLogs, iisw3c.ConsolidateLogs);
Assert.IsNull(iisw3c.ICheckpoint);
}
[Test]
public void ParametersOfGrok()
{
string match = "%{IPAddress:ip1} %{IPAddress:ip2}";
TimberWinR.Configuration.Pair addField = new TimberWinR.Configuration.Pair("field1", @"%{foo}");
bool dropIfMatch = true;
string removeField = "ip1";
TimberWinR.Filters.GrokFilter grok = c.Groks.ToArray()[0];
Assert.AreEqual(match, grok.Match);
Assert.AreEqual(addField, grok.AddField);
Assert.AreEqual(dropIfMatch, grok.DropIfMatch);
Assert.AreEqual(removeField, grok.RemoveField);
}
}
}
}

8
TimberWinR/Configuration.cs
View File

@@ -116,11 +116,11 @@ namespace TimberWinR
get { return _iisw3clogs; }
}
private static List<GrokFilter> _groks = new List<GrokFilter>();
private static List<FilterBase> _filters = new List<FilterBase>();
public IEnumerable<GrokFilter> Groks
public IEnumerable<FilterBase> Filters
{
get { return _groks; }
get { return _filters; }
}
public Configuration(string xmlConfFile)
@@ -478,7 +478,7 @@ namespace TimberWinR
case "Grok":
Params_Grok args = parseParams_Grok(e.Elements());
GrokFilter grok = new GrokFilter(args);
_groks.Add(grok);
_filters.Add(grok);
break;
case "Mutate":
break;

2
TimberWinR/Filters/FilterBase.cs
View File

@@ -8,6 +8,6 @@ namespace TimberWinR.Filters
{
public abstract class FilterBase
{
public abstract void Apply(JObject json);
public abstract void Apply(JObject json);
}
}

50
TimberWinR/Filters/GrokFilter.cs
View File

@@ -1,7 +1,11 @@
using System;
using Newtonsoft.Json.Linq;
using RapidRegex.Core;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace TimberWinR.Filters
{
@@ -39,8 +43,46 @@ namespace TimberWinR.Filters
public override void Apply(Newtonsoft.Json.Linq.JObject json)
{
throw new NotImplementedException();
JToken token = null;
if (json.TryGetValue(Field, StringComparison.OrdinalIgnoreCase, out token))
{
string text = token.ToString();
if (!string.IsNullOrEmpty(text))
{
string expr = Match;
var resolver = new RegexGrokResolver();
var pattern = resolver.ResolveToRegex(expr);
var match = Regex.Match(text, pattern);
if (match.Success)
{
var regex = new Regex(pattern);
var namedCaptures = regex.MatchNamedCaptures(text);
foreach (string fieldName in namedCaptures.Keys)
{
if (fieldName == "timestamp")
{
string value = namedCaptures[fieldName];
DateTime ts;
if (DateTime.TryParse(value, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else if (DateTime.TryParseExact(value, new string[]
{
"MMM dd hh:mm:ss",
"MMM dd HH:mm:ss",
"MMM dd h:mm",
"MMM dd hh:mm",
}, CultureInfo.InvariantCulture, DateTimeStyles.None, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else
json.Add(fieldName, (JToken) namedCaptures[fieldName]);
}
else
json.Add(fieldName, (JToken) namedCaptures[fieldName]);
}
}
}
}
}
}
}
}

45
TimberWinR/Outputs/Redis.cs
View File

@@ -80,7 +80,7 @@ namespace TimberWinR.Outputs
/// <param name="jsonMessage"></param>
protected override void MessageReceivedHandler(JObject jsonMessage)
{
if (_manager.Config.Groks != null)
if (_manager.Config.Filters != null)
ProcessGroks(jsonMessage);
var message = jsonMessage.ToString();
@@ -94,48 +94,9 @@ namespace TimberWinR.Outputs
private void ProcessGroks(JObject json)
{
foreach (var grok in _manager.Config.Groks)
foreach (var grok in _manager.Config.Filters)
{
JToken token = null;
if (json.TryGetValue(grok.Field, StringComparison.OrdinalIgnoreCase, out token))
{
string text = token.ToString();
if (!string.IsNullOrEmpty(text))
{
string expr = grok.Match;
var resolver = new RegexGrokResolver();
var pattern = resolver.ResolveToRegex(expr);
var match = Regex.Match(text, pattern);
if (match.Success)
{
var regex = new Regex(pattern);
var namedCaptures = regex.MatchNamedCaptures(text);
foreach (string fieldName in namedCaptures.Keys)
{
if (fieldName == "timestamp")
{
string value = namedCaptures[fieldName];
DateTime ts;
if (DateTime.TryParse(value, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else if (DateTime.TryParseExact(value, new string[]
{
"MMM dd hh:mm:ss",
"MMM dd HH:mm:ss",
"MMM dd h:mm",
"MMM dd hh:mm",
}, CultureInfo.InvariantCulture, DateTimeStyles.None, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else
json.Add(fieldName, (JToken)namedCaptures[fieldName]);
}
else
json.Add(fieldName, (JToken)namedCaptures[fieldName]);
}
}
}
}
grok.Apply(json);
}
}