TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated to Json format

Browse Source
This commit is contained in:
Eric Fontana
2014-07-25 14:39:55 -04:00
parent 2f8e3694eb
commit debf0cf553
8 changed files with 356 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
TimberWinR.UnitTests/GrokFilterTests.cs Normal file
View File

@@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using NUnit.Framework;
using TimberWinR.Parser;
using Newtonsoft.Json.Linq;
namespace TimberWinR.UnitTests
{
[TestFixture]
public class GrokFilterTests
{
[Test]
public void TestMatch()
{
JObject json = new JObject
{
{"LogFilename", @"C:\\Logs1\\test1.log"},
{"Index", 7},
{"Text", null},
{"type", "Win32-FileLog"},
{"ComputerName", "dev.vistaprint.net"}
};
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
],
""add_field"":[
""host"",
""%{ComputerName}""
]
}
}]
}
}";
Configuration c = Configuration.FromString(grokJson);
Grok grok = c.Filters.First() as Grok;
Assert.IsTrue(grok.Apply(json));
Assert.AreEqual(json["host"].ToString(), "dev.vistaprint.net");
}
}
}

5
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -30,6 +30,10 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="nunit.framework">
<HintPath>..\packages\NUnit.2.6.3\lib\nunit.framework.dll</HintPath>
</Reference>
@@ -43,6 +47,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="Configuration.cs" />
<Compile Include="GrokFilterTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>

1
TimberWinR.UnitTests/packages.config
View File

@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="NUnit" version="2.6.3" targetFramework="net45" />
</packages>

248
TimberWinR/Filters/GrokFilter.cs
View File

@@ -1,112 +1,111 @@
using Newtonsoft.Json.Linq;
using RapidRegex.Core;
using System;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Threading;
using Microsoft.CSharp;
using Newtonsoft.Json.Linq;
using NLog;
using RapidRegex.Core;
namespace TimberWinR.Filters
namespace TimberWinR.Parser
{
public class GrokFilter : FilterBase
public class Fields
{
public new const string TagName = "Grok";
private Dictionary<string, string> fields { get; set; }
public string Match { get; private set; }
public string Field { get; private set; }
public List<FieldValuePair> AddFields { get; private set; }
public bool DropIfMatch { get; private set; }
public List<string> RemoveFields { get; private set; }
public List<AddTag> AddTags { get; private set; }
public static void Parse(List<FilterBase> filters, XElement grokElement)
public string this[string i]
{
filters.Add(parseGrok(grokElement));
get { return fields[i]; }
set { fields[i] = value; }
}
static GrokFilter parseGrok(XElement e)
public Fields(JObject json)
{
return new GrokFilter(e);
fields = new Dictionary<string, string>();
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
fields[key] = json[key].ToString();
}
}
GrokFilter(XElement parent)
public partial class Grok : LogstashFilter
{
public override bool Apply(JObject json)
{
AddTags = new List<AddTag>();
AddFields = new List<FieldValuePair>();
RemoveFields = new List<string>();
if (Condition != null && !EvaluateCondition(json))
return false;
ParseMatch(parent);
ParseAddFields(parent);
ParseAddTags(parent);
ParseDropIfMatch(parent);
ParseRemoveFields(parent);
}
private void ParseMatch(XElement parent)
{
XElement e = parent.Element("Match");
Field = e.Attribute("field").Value;
Match = e.Attribute("value").Value;
}
private void ParseAddFields(XElement parent)
{
foreach (var e in parent.Elements("AddField"))
{
AddFields.Add(new FieldValuePair(ParseStringAttribute(e, "field"), ParseStringAttribute(e, "value")));
}
}
private void ParseDropIfMatch(XElement parent)
{
XElement e = parent.Element("DropIfMatch");
if (e != null)
if (Matches(json))
{
DropIfMatch = ParseBoolAttribute(e, "value", false);
AddFields(json);
AddTags(json);
return true;
}
return false;
}
private void ParseRemoveFields(XElement parent)
private bool EvaluateCondition(JObject json)
{
foreach (var e in parent.Elements("RemoveField"))
// Create a new instance of the C# compiler
var cond = Condition;
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString()));
var compiler = new CSharpCodeProvider();
// Create some parameters for the compiler
var parms = new System.CodeDom.Compiler.CompilerParameters
{
if (e != null)
{
RemoveFields.Add(ParseStringAttribute(e, "value"));
}
GenerateExecutable = false,
GenerateInMemory = true
};
parms.ReferencedAssemblies.Add("System.dll");
var code = string.Format(@" using System;
class EvaluatorClass
{{
public bool Evaluate()
{{
return {0};
}}
}}", cond);
// Try to compile the string into an assembly
var results = compiler.CompileAssemblyFromSource(parms, new string[] { code });
// If there weren't any errors get an instance of "MyClass" and invoke
// the "Message" method on it
if (results.Errors.Count == 0)
{
var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass");
var result = evClass.GetType().
GetMethod("Evaluate").
Invoke(evClass, null);
return bool.Parse(result.ToString());
}
}
/// <summary>
/// Apply the Grok filter to the Object
/// </summary>
/// <param name="json"></param>
public override void Apply(Newtonsoft.Json.Linq.JObject json)
{
if (ApplyMatch(json))
else
{
foreach (var at in AddTags)
at.Apply(json);
}
foreach (var e in results.Errors)
LogManager.GetCurrentClassLogger().Error(e);
}
return false;
}
/// <summary>
/// Apply the Match filter, if there is none specified, it's considered a match.
/// </summary>
/// <param name="json"></param>
/// <returns></returns>
private bool ApplyMatch(Newtonsoft.Json.Linq.JObject json)
{
private bool Matches(Newtonsoft.Json.Linq.JObject json)
{
string field = Match[0];
string expr = Match[1];
JToken token = null;
if (json.TryGetValue(Field, StringComparison.OrdinalIgnoreCase, out token))
if (json.TryGetValue(field, out token))
{
string text = token.ToString();
if (!string.IsNullOrEmpty(text))
{
string expr = Match;
var resolver = new RegexGrokResolver();
var pattern = resolver.ResolveToRegex(expr);
var match = Regex.Match(text, pattern);
@@ -121,87 +120,42 @@ namespace TimberWinR.Filters
return true; // Yes!
}
}
return false; // Empty field is no match
return true; // Empty field is no match
}
return true; // Not specified is success
return false; // Not specified is failure
}
private void AddOrModify(JObject json, string fieldName, string fieldValue)
private void AddFields(Newtonsoft.Json.Linq.JObject json)
{
if (json[fieldName] == null)
json.Add(fieldName, fieldValue);
else
json[fieldName] = fieldValue;
}
public class FieldValuePair
{
public string Field { get; set; }
public string Value { get; set; }
public FieldValuePair(string field, string value)
if (AddField != null && AddField.Length > 0)
{
Field = field;
Value = value;
}
}
private void ParseAddTags(XElement parent)
{
foreach (var e in parent.Elements("AddTag"))
{
AddTags.Add(new AddTag(e));
}
}
public class AddTag
{
public string Value { get; set; }
public AddTag(XElement e)
{
Value = e.Value.Trim();
}
public void Apply(Newtonsoft.Json.Linq.JObject json)
{
string value = ReplaceTokens(Value, json);
JToken tags = json["tags"];
if (tags == null)
json.Add("tags", new JArray(value));
else
for (int i = 0; i < AddField.Length; i += 2)
{
JArray a = tags as JArray;
a.Add(value);
string fieldName = ExpandField(AddField[i], json);
string fieldValue = ExpandField(AddField[i + 1], json);
AddOrModify(json, fieldName, fieldValue);
}
}
}
private string ReplaceTokens(string fieldName, JObject json)
private void AddTags(Newtonsoft.Json.Linq.JObject json)
{
if (AddTag != null && AddTag.Length > 0)
{
foreach (var token in json.Children())
for (int i = 0; i < AddTag.Length; i++)
{
string replaceString = "%{" + token.Path + "}";
fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
string value = ExpandField(AddTag[i], json);
JToken tags = json["tags"];
if (tags == null)
json.Add("tags", new JArray(value));
else
{
JArray a = tags as JArray;
a.Add(value);
}
}
return fieldName;
}
}
}
public struct Pair
{
public readonly string Name, Value;
public Pair(string name, string value)
{
Name = name;
Value = value;
}
public override string ToString()
{
return String.Format("Name:= {0} , Value:= {1}", Name, Value);
}
}
}
}

168
TimberWinR/Filters/MutateFilter.cs
View File

@@ -7,120 +7,70 @@ using System.Xml.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace TimberWinR.Filters
namespace TimberWinR.Parser
{
public class MutateFilter : FilterBase
{
public new const string TagName = "Mutate";
public partial class Mutate : LogstashFilter
{
public override bool Apply(JObject json)
{
ApplySplits(json);
ApplyRenames(json);
ApplyReplace(json);
return true;
}
public List<MutateOperation> Operations { get; private set; }
private void ApplyRenames(JObject json)
{
if (Rename != null && Rename.Length > 0)
{
for (int i = 0; i < Rename.Length; i += 2)
{
string oldName = ExpandField(Rename[i], json);
string newName = ExpandField(Rename[i + 1], json);
RenameProperty(json, oldName, newName);
}
}
}
public static void Parse(List<FilterBase> filters, XElement mutateElement)
{
filters.Add(parseMutate(mutateElement));
}
static MutateFilter parseMutate(XElement e)
{
return new MutateFilter(e);
}
private void ApplySplits(JObject json)
{
if (Split != null && Split.Length > 0)
{
for (int i = 0; i < Split.Length; i += 2)
{
string fieldName = Split[i];
string splitChar = Split[i + 1];
MutateFilter(XElement parent)
{
Operations = new List<MutateOperation>();
JArray array = null;
if (json[fieldName] != null)
{
string valueToSplit = json[fieldName].ToString();
string[] values = valueToSplit.Split(new string[] {splitChar}, StringSplitOptions.None);
foreach (string value in values)
{
if (array == null)
array = new JArray(value);
else
array.Add(value);
}
ParseOperations(parent);
}
}
}
}
}
private void ParseOperations(XElement parent)
{
foreach (var e in parent.Elements())
{
switch(e.Name.ToString())
{
case Rename.TagName:
ParseRename(e);
break;
case Remove.TagName:
ParseRemove(e);
break;
}
}
}
public override void Apply(JObject json)
{
foreach (var r in Operations)
r.Apply(json);
}
public abstract class MutateOperation
{
public abstract void Apply(JObject json);
}
private void ParseRemove(XElement e)
{
var o = new Remove(e.Attribute("field").Value);
Operations.Add(o);
}
class Remove : MutateOperation
{
public const string TagName = "Remove";
public string FieldName { get; set; }
public Remove(string fieldName)
{
FieldName = fieldName;
}
public override void Apply(JObject json)
{
var fieldName = FieldName;
if (fieldName.Contains('%'))
fieldName = ReplaceTokens(fieldName, json);
json.Remove(fieldName);
}
private string ReplaceTokens(string fieldName, JObject json)
{
foreach(var token in json.Children())
{
string replaceString = "%{" + token.Path + "}";
fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
}
return fieldName;
}
}
private void ParseRename(XElement e)
{
var o = new Rename(e.Attribute("oldName").Value, e.Attribute("newName").Value);
Operations.Add(o);
}
class Rename : MutateOperation
{
public const string TagName = "Rename";
public string OldName { get; set; }
public string NewName { get; set; }
public Rename(string oldName, string newName)
{
OldName = oldName;
NewName = newName;
}
public override void Apply(JObject json)
{
JToken token = json[OldName];
if (token != null)
{
json.Remove(OldName);
json.Add(NewName, token);
}
}
}
}
private void ApplyReplace(JObject json)
{
if (Replace != null && Replace.Length > 0)
{
for (int i = 0; i < Replace.Length; i += 2)
{
string fieldName = Replace[0];
string replaceValue = ExpandField(Replace[i + 1], json);
ReplaceProperty(json, fieldName, replaceValue);
}
}
}
}
}

118
TimberWinR/Inputs/InputBase.cs
View File

@@ -8,11 +8,11 @@ using System.Xml.Linq;
namespace TimberWinR.Inputs
{
public abstract class InputBase : Parsers
public abstract class InputBase
{
public const string TagName = "Inputs";
protected static List<FieldDefinition> ParseFields(XElement parent, Dictionary<string, Type> allPossibleFields)
internal List<FieldDefinition> parseFields(XElement parent, Dictionary<string, Type> allPossibleFields)
{
IEnumerable<XElement> xml_fields =
from el in parent.Elements("Fields").Elements("Field")
@@ -57,6 +57,120 @@ namespace TimberWinR.Inputs
return fields;
}
protected static string ParseRequiredStringAttribute(XElement e, string attributeName)
{
XAttribute a = e.Attribute(attributeName);
if (a != null)
return a.Value;
else
throw new TimberWinR.ConfigurationErrors.MissingRequiredAttributeException(e, attributeName);
}
protected static string ParseStringAttribute(XElement e, string attributeName, string defaultValue = "")
{
string retValue = defaultValue;
XAttribute a = e.Attribute(attributeName);
if (a != null)
retValue = a.Value;
return retValue;
}
protected static string ParseDateAttribute(XElement e, string attributeName, string defaultValue = "")
{
string retValue = defaultValue;
XAttribute a = e.Attribute(attributeName);
if (a != null)
{
DateTime dt;
if (DateTime.TryParseExact(a.Value,
"yyyy-MM-dd hh:mm:ss",
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out dt))
{
retValue = a.Value;
}
else
{
throw new TimberWinR.ConfigurationErrors.InvalidAttributeDateValueException(a);
}
}
return retValue;
}
protected static bool ParseRequiredBoolAttribute(XElement e, string attributeName)
{
XAttribute a = e.Attribute(attributeName);
if (a == null)
throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName));
switch (a.Value)
{
case "ON":
case "true":
return true;
case "OFF":
case "false":
return false;
default:
throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName));
}
}
protected static string ParseEnumAttribute(XElement e, string attributeName, IEnumerable<string> values, string defaultValue)
{
XAttribute a = e.Attribute(attributeName);
if (a != null)
{
string v = a.Value;
if (values.Contains(v))
return v;
else
throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName));
}
return defaultValue;
}
protected static int ParseIntAttribute(XElement e, string attributeName, int defaultValue)
{
XAttribute a = e.Attribute(attributeName);
if (a != null)
{
int valInt;
if (int.TryParse(a.Value, out valInt))
return valInt;
else
throw new TimberWinR.ConfigurationErrors.InvalidAttributeIntegerValueException(a);
}
return defaultValue;
}
protected static bool ParseBoolAttribute(XElement e, string attributeName, bool defaultValue)
{
bool retValue = defaultValue;
XAttribute a = e.Attribute(attributeName);
if (a != null)
{
switch (a.Value)
{
case "ON":
case "true":
retValue = true;
break;
case "OFF":
case "false":
retValue = false;
break;
}
}
return retValue;
}
public override string ToString()
{
StringBuilder sb = new StringBuilder();

29
TimberWinR/Inputs/TailFileInputListener.cs
View File

@@ -23,9 +23,9 @@ namespace TimberWinR.Inputs
public class TailFileInputListener : InputListener
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Inputs.TailFileInput _arguments;
private TimberWinR.Parser.Log _arguments;
public TailFileInputListener(TimberWinR.Inputs.TailFileInput arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
public TailFileInputListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
{
_arguments = arguments;
@@ -35,15 +35,13 @@ namespace TimberWinR.Inputs
}
private void FileWatcher()
{
{
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
var iFmt = new TextLineInputFormat()
{
iCodepage = _arguments.ICodepage,
iCodepage = _arguments.CodePage,
splitLongLines = _arguments.SplitLongLines,
iCheckpoint = checkpointFileName,
recurse = _arguments.Recurse
@@ -52,6 +50,12 @@ namespace TimberWinR.Inputs
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", ".")
.ToString();
var firstQuery = true;
// Execute the query
while (!CancelToken.IsCancellationRequested)
@@ -81,13 +85,16 @@ namespace TimberWinR.Inputs
continue;
object v = record.getValue(field.Name);
if (field.FieldType == typeof(DateTime))
v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
if (field.DataType == typeof(DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-FileLog"));
json.Add(new JProperty("ComputerName", computerName));
ProcessJson(json);
}
}

1
TimberWinR/TimberWinR.csproj
View File

@@ -81,7 +81,6 @@
<Compile Include="Outputs\OutputSender.cs" />
<Compile Include="Outputs\Redis.cs" />
<Compile Include="Parser.cs" />
<Compile Include="Parsers.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Properties\Resources.Designer.cs">
<AutoGen>True</AutoGen>