diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs new file mode 100644 index 0000000..d3a9653 --- /dev/null +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using NUnit.Framework; +using TimberWinR.Parser; +using Newtonsoft.Json.Linq; + +namespace TimberWinR.UnitTests +{ + [TestFixture] + public class GrokFilterTests + { + [Test] + public void TestMatch() + { + JObject json = new JObject + { + {"LogFilename", @"C:\\Logs1\\test1.log"}, + {"Index", 7}, + {"Text", null}, + {"type", "Win32-FileLog"}, + {"ComputerName", "dev.vistaprint.net"} + }; + + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", + """" + ], + ""add_field"":[ + ""host"", + ""%{ComputerName}"" + ] + } + }] + } + }"; + + Configuration c = Configuration.FromString(grokJson); + + Grok grok = c.Filters.First() as Grok; + + Assert.IsTrue(grok.Apply(json)); + + Assert.AreEqual(json["host"].ToString(), "dev.vistaprint.net"); + } + } +} diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj index 540df1c..44a94a8 100644 --- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj +++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj @@ -30,6 +30,10 @@ 4 + + False + ..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll + ..\packages\NUnit.2.6.3\lib\nunit.framework.dll @@ -43,6 +47,7 @@ + diff --git a/TimberWinR.UnitTests/packages.config b/TimberWinR.UnitTests/packages.config index ad37a52..22e7fbb 100644 --- a/TimberWinR.UnitTests/packages.config +++ b/TimberWinR.UnitTests/packages.config @@ -1,4 +1,5 @@  + \ No newline at end of file diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index 9a439b2..37d0045 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -1,112 +1,111 @@ -using Newtonsoft.Json.Linq; -using RapidRegex.Core; -using System; +using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Text; using System.Text.RegularExpressions; -using System.Xml.Linq; +using System.Threading; +using Microsoft.CSharp; +using Newtonsoft.Json.Linq; +using NLog; +using RapidRegex.Core; -namespace TimberWinR.Filters +namespace TimberWinR.Parser { - public class GrokFilter : FilterBase + public class Fields { - public new const string TagName = "Grok"; + private Dictionary fields { get; set; } - public string Match { get; private set; } - public string Field { get; private set; } - public List AddFields { get; private set; } - public bool DropIfMatch { get; private set; } - public List RemoveFields { get; private set; } - public List AddTags { get; private set; } - - public static void Parse(List filters, XElement grokElement) + public string this[string i] { - filters.Add(parseGrok(grokElement)); + get { return fields[i]; } + set { fields[i] = value; } } - static GrokFilter parseGrok(XElement e) + public Fields(JObject json) { - return new GrokFilter(e); + fields = new Dictionary(); + IList keys = json.Properties().Select(p => p.Name).ToList(); + foreach (string key in keys) + fields[key] = json[key].ToString(); } + } - GrokFilter(XElement parent) + public partial class Grok : LogstashFilter + { + public override bool Apply(JObject json) { - AddTags = new List(); - AddFields = new List(); - RemoveFields = new List(); + if (Condition != null && !EvaluateCondition(json)) + return false; - ParseMatch(parent); - ParseAddFields(parent); - ParseAddTags(parent); - ParseDropIfMatch(parent); - ParseRemoveFields(parent); - } - - private void ParseMatch(XElement parent) - { - XElement e = parent.Element("Match"); - Field = e.Attribute("field").Value; - Match = e.Attribute("value").Value; - } - - private void ParseAddFields(XElement parent) - { - foreach (var e in parent.Elements("AddField")) - { - AddFields.Add(new FieldValuePair(ParseStringAttribute(e, "field"), ParseStringAttribute(e, "value"))); - } - } - - private void ParseDropIfMatch(XElement parent) - { - XElement e = parent.Element("DropIfMatch"); - - if (e != null) + if (Matches(json)) { - DropIfMatch = ParseBoolAttribute(e, "value", false); + AddFields(json); + AddTags(json); + return true; } + return false; } - private void ParseRemoveFields(XElement parent) + private bool EvaluateCondition(JObject json) { - foreach (var e in parent.Elements("RemoveField")) + // Create a new instance of the C# compiler + var cond = Condition; + + IList keys = json.Properties().Select(p => p.Name).ToList(); + foreach (string key in keys) + cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); + + var compiler = new CSharpCodeProvider(); + + // Create some parameters for the compiler + var parms = new System.CodeDom.Compiler.CompilerParameters { - if (e != null) - { - RemoveFields.Add(ParseStringAttribute(e, "value")); - } + GenerateExecutable = false, + GenerateInMemory = true + }; + parms.ReferencedAssemblies.Add("System.dll"); + var code = string.Format(@" using System; + class EvaluatorClass + {{ + public bool Evaluate() + {{ + return {0}; + }} + }}", cond); + + // Try to compile the string into an assembly + var results = compiler.CompileAssemblyFromSource(parms, new string[] { code }); + + // If there weren't any errors get an instance of "MyClass" and invoke + // the "Message" method on it + if (results.Errors.Count == 0) + { + var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass"); + var result = evClass.GetType(). + GetMethod("Evaluate"). + Invoke(evClass, null); + return bool.Parse(result.ToString()); } - } - - /// - /// Apply the Grok filter to the Object - /// - /// - public override void Apply(Newtonsoft.Json.Linq.JObject json) - { - if (ApplyMatch(json)) + else { - foreach (var at in AddTags) - at.Apply(json); - } + foreach (var e in results.Errors) + LogManager.GetCurrentClassLogger().Error(e); + } + + return false; } - /// - /// Apply the Match filter, if there is none specified, it's considered a match. - /// - /// - /// - private bool ApplyMatch(Newtonsoft.Json.Linq.JObject json) - { + private bool Matches(Newtonsoft.Json.Linq.JObject json) + { + string field = Match[0]; + string expr = Match[1]; + JToken token = null; - if (json.TryGetValue(Field, StringComparison.OrdinalIgnoreCase, out token)) + if (json.TryGetValue(field, out token)) { string text = token.ToString(); if (!string.IsNullOrEmpty(text)) { - string expr = Match; var resolver = new RegexGrokResolver(); var pattern = resolver.ResolveToRegex(expr); var match = Regex.Match(text, pattern); @@ -121,87 +120,42 @@ namespace TimberWinR.Filters return true; // Yes! } } - return false; // Empty field is no match + return true; // Empty field is no match } - return true; // Not specified is success + return false; // Not specified is failure } - private void AddOrModify(JObject json, string fieldName, string fieldValue) + private void AddFields(Newtonsoft.Json.Linq.JObject json) { - if (json[fieldName] == null) - json.Add(fieldName, fieldValue); - else - json[fieldName] = fieldValue; - } - - public class FieldValuePair - { - public string Field { get; set; } - public string Value { get; set; } - - public FieldValuePair(string field, string value) + if (AddField != null && AddField.Length > 0) { - Field = field; - Value = value; - } - } - - private void ParseAddTags(XElement parent) - { - foreach (var e in parent.Elements("AddTag")) - { - AddTags.Add(new AddTag(e)); - } - } - - public class AddTag - { - public string Value { get; set; } - public AddTag(XElement e) - { - Value = e.Value.Trim(); - } - - public void Apply(Newtonsoft.Json.Linq.JObject json) - { - string value = ReplaceTokens(Value, json); - JToken tags = json["tags"]; - if (tags == null) - json.Add("tags", new JArray(value)); - else + for (int i = 0; i < AddField.Length; i += 2) { - JArray a = tags as JArray; - a.Add(value); + string fieldName = ExpandField(AddField[i], json); + string fieldValue = ExpandField(AddField[i + 1], json); + AddOrModify(json, fieldName, fieldValue); } } + } - private string ReplaceTokens(string fieldName, JObject json) + private void AddTags(Newtonsoft.Json.Linq.JObject json) + { + if (AddTag != null && AddTag.Length > 0) { - foreach (var token in json.Children()) + for (int i = 0; i < AddTag.Length; i++) { - string replaceString = "%{" + token.Path + "}"; - fieldName = fieldName.Replace(replaceString, json[token.Path].ToString()); + string value = ExpandField(AddTag[i], json); + + JToken tags = json["tags"]; + if (tags == null) + json.Add("tags", new JArray(value)); + else + { + JArray a = tags as JArray; + a.Add(value); + } } - return fieldName; } - - } - } - - public struct Pair - { - public readonly string Name, Value; - - public Pair(string name, string value) - { - Name = name; - Value = value; - } - - public override string ToString() - { - return String.Format("Name:= {0} , Value:= {1}", Name, Value); } } - -} \ No newline at end of file +} diff --git a/TimberWinR/Filters/MutateFilter.cs b/TimberWinR/Filters/MutateFilter.cs index 7bfdc2a..55f71f9 100644 --- a/TimberWinR/Filters/MutateFilter.cs +++ b/TimberWinR/Filters/MutateFilter.cs @@ -7,120 +7,70 @@ using System.Xml.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace TimberWinR.Filters +namespace TimberWinR.Parser { - public class MutateFilter : FilterBase - { - public new const string TagName = "Mutate"; + public partial class Mutate : LogstashFilter + { + public override bool Apply(JObject json) + { + ApplySplits(json); + ApplyRenames(json); + ApplyReplace(json); + return true; + } - public List Operations { get; private set; } + private void ApplyRenames(JObject json) + { + if (Rename != null && Rename.Length > 0) + { + for (int i = 0; i < Rename.Length; i += 2) + { + string oldName = ExpandField(Rename[i], json); + string newName = ExpandField(Rename[i + 1], json); + RenameProperty(json, oldName, newName); + } + } + } - public static void Parse(List filters, XElement mutateElement) - { - filters.Add(parseMutate(mutateElement)); - } - - static MutateFilter parseMutate(XElement e) - { - return new MutateFilter(e); - } + private void ApplySplits(JObject json) + { + if (Split != null && Split.Length > 0) + { + for (int i = 0; i < Split.Length; i += 2) + { + string fieldName = Split[i]; + string splitChar = Split[i + 1]; - MutateFilter(XElement parent) - { - Operations = new List(); + JArray array = null; + if (json[fieldName] != null) + { + string valueToSplit = json[fieldName].ToString(); + string[] values = valueToSplit.Split(new string[] {splitChar}, StringSplitOptions.None); + foreach (string value in values) + { + if (array == null) + array = new JArray(value); + else + array.Add(value); + } - ParseOperations(parent); - } + } + } + } + } - private void ParseOperations(XElement parent) - { - foreach (var e in parent.Elements()) - { - switch(e.Name.ToString()) - { - case Rename.TagName: - ParseRename(e); - break; - case Remove.TagName: - ParseRemove(e); - break; - } - } - } - public override void Apply(JObject json) - { - foreach (var r in Operations) - r.Apply(json); - } - - public abstract class MutateOperation - { - public abstract void Apply(JObject json); - } - - private void ParseRemove(XElement e) - { - var o = new Remove(e.Attribute("field").Value); - Operations.Add(o); - } - - class Remove : MutateOperation - { - public const string TagName = "Remove"; - public string FieldName { get; set; } - - public Remove(string fieldName) - { - FieldName = fieldName; - } - - public override void Apply(JObject json) - { - var fieldName = FieldName; - if (fieldName.Contains('%')) - fieldName = ReplaceTokens(fieldName, json); - json.Remove(fieldName); - } - - private string ReplaceTokens(string fieldName, JObject json) - { - foreach(var token in json.Children()) - { - string replaceString = "%{" + token.Path + "}"; - fieldName = fieldName.Replace(replaceString, json[token.Path].ToString()); - } - return fieldName; - } - } - - private void ParseRename(XElement e) - { - var o = new Rename(e.Attribute("oldName").Value, e.Attribute("newName").Value); - Operations.Add(o); - } - - class Rename : MutateOperation - { - public const string TagName = "Rename"; - public string OldName { get; set; } - public string NewName { get; set; } - - public Rename(string oldName, string newName) - { - OldName = oldName; - NewName = newName; - } - - public override void Apply(JObject json) - { - JToken token = json[OldName]; - if (token != null) - { - json.Remove(OldName); - json.Add(NewName, token); - } - } - } - } + private void ApplyReplace(JObject json) + { + if (Replace != null && Replace.Length > 0) + { + for (int i = 0; i < Replace.Length; i += 2) + { + string fieldName = Replace[0]; + string replaceValue = ExpandField(Replace[i + 1], json); + ReplaceProperty(json, fieldName, replaceValue); + } + } + } + } } diff --git a/TimberWinR/Inputs/InputBase.cs b/TimberWinR/Inputs/InputBase.cs index 9b4e116..a5dfea6 100644 --- a/TimberWinR/Inputs/InputBase.cs +++ b/TimberWinR/Inputs/InputBase.cs @@ -8,11 +8,11 @@ using System.Xml.Linq; namespace TimberWinR.Inputs { - public abstract class InputBase : Parsers + public abstract class InputBase { public const string TagName = "Inputs"; - protected static List ParseFields(XElement parent, Dictionary allPossibleFields) + internal List parseFields(XElement parent, Dictionary allPossibleFields) { IEnumerable xml_fields = from el in parent.Elements("Fields").Elements("Field") @@ -57,6 +57,120 @@ namespace TimberWinR.Inputs return fields; } + protected static string ParseRequiredStringAttribute(XElement e, string attributeName) + { + XAttribute a = e.Attribute(attributeName); + if (a != null) + return a.Value; + else + throw new TimberWinR.ConfigurationErrors.MissingRequiredAttributeException(e, attributeName); + } + + protected static string ParseStringAttribute(XElement e, string attributeName, string defaultValue = "") + { + string retValue = defaultValue; + XAttribute a = e.Attribute(attributeName); + if (a != null) + retValue = a.Value; + return retValue; + } + + protected static string ParseDateAttribute(XElement e, string attributeName, string defaultValue = "") + { + string retValue = defaultValue; + XAttribute a = e.Attribute(attributeName); + if (a != null) + { + DateTime dt; + if (DateTime.TryParseExact(a.Value, + "yyyy-MM-dd hh:mm:ss", + CultureInfo.InvariantCulture, + DateTimeStyles.None, + out dt)) + { + retValue = a.Value; + } + else + { + throw new TimberWinR.ConfigurationErrors.InvalidAttributeDateValueException(a); + } + } + + return retValue; + } + + protected static bool ParseRequiredBoolAttribute(XElement e, string attributeName) + { + XAttribute a = e.Attribute(attributeName); + if (a == null) + throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); + + switch (a.Value) + { + case "ON": + case "true": + return true; + + case "OFF": + case "false": + return false; + + default: + throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); + } + } + + protected static string ParseEnumAttribute(XElement e, string attributeName, IEnumerable values, string defaultValue) + { + XAttribute a = e.Attribute(attributeName); + + if (a != null) + { + string v = a.Value; + if (values.Contains(v)) + return v; + else + throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); + } + return defaultValue; + } + + protected static int ParseIntAttribute(XElement e, string attributeName, int defaultValue) + { + XAttribute a = e.Attribute(attributeName); + if (a != null) + { + int valInt; + if (int.TryParse(a.Value, out valInt)) + return valInt; + else + throw new TimberWinR.ConfigurationErrors.InvalidAttributeIntegerValueException(a); + } + return defaultValue; + } + protected static bool ParseBoolAttribute(XElement e, string attributeName, bool defaultValue) + { + bool retValue = defaultValue; + XAttribute a = e.Attribute(attributeName); + + if (a != null) + { + switch (a.Value) + { + case "ON": + case "true": + retValue = true; + break; + + case "OFF": + case "false": + retValue = false; + break; + } + } + return retValue; + } + public override string ToString() { StringBuilder sb = new StringBuilder(); diff --git a/TimberWinR/Inputs/TailFileInputListener.cs b/TimberWinR/Inputs/TailFileInputListener.cs index 583b813..c04e390 100644 --- a/TimberWinR/Inputs/TailFileInputListener.cs +++ b/TimberWinR/Inputs/TailFileInputListener.cs @@ -23,9 +23,9 @@ namespace TimberWinR.Inputs public class TailFileInputListener : InputListener { private int _pollingIntervalInSeconds = 1; - private TimberWinR.Inputs.TailFileInput _arguments; + private TimberWinR.Parser.Log _arguments; - public TailFileInputListener(TimberWinR.Inputs.TailFileInput arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) + public TailFileInputListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken) { _arguments = arguments; @@ -35,15 +35,13 @@ namespace TimberWinR.Inputs } private void FileWatcher() - { - - + { var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString())); var iFmt = new TextLineInputFormat() { - iCodepage = _arguments.ICodepage, + iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, iCheckpoint = checkpointFileName, recurse = _arguments.Recurse @@ -52,6 +50,12 @@ namespace TimberWinR.Inputs // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Location); + string computerName = System.Environment.MachineName + "." + + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( + @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") + .GetValue("Domain", ".") + .ToString(); + var firstQuery = true; // Execute the query while (!CancelToken.IsCancellationRequested) @@ -81,13 +85,16 @@ namespace TimberWinR.Inputs continue; object v = record.getValue(field.Name); - - if (field.FieldType == typeof(DateTime)) - v = field.ToDateTime(v).ToUniversalTime(); - - json.Add(new JProperty(field.Name, v)); + if (field.DataType == typeof(DateTime)) + { + DateTime dt = DateTime.Parse(v.ToString()); + json.Add(new JProperty(field.Name, dt)); + } + else + json.Add(new JProperty(field.Name, v)); } json.Add(new JProperty("type", "Win32-FileLog")); + json.Add(new JProperty("ComputerName", computerName)); ProcessJson(json); } } diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index df99707..9da6995 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -81,7 +81,6 @@ - True