From 65de7cbc9355c12080d93ed784e7a0a89a55fd16 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Mon, 28 Jul 2014 10:55:17 -0400 Subject: [PATCH] Added Validate method Added tests to validate the configuration. --- TimberWinR.ServiceHost/Program.cs | 9 +- TimberWinR.ServiceHost/config.json | 186 ++++++++++------- TimberWinR.UnitTests/Configuration.cs | 60 +++++- TimberWinR.UnitTests/GrokFilterTests.cs | 1 - TimberWinR/Configuration.cs | 47 ++++- TimberWinR/Filters/DateFilter.cs | 96 ++++----- TimberWinR/Filters/EvaluatingFilter.cs | 65 ++++++ TimberWinR/Filters/GrokFilter.cs | 51 +---- TimberWinR/Filters/MutateFilter.cs | 3 + TimberWinR/Inputs/IISW3CInputListener.cs | 5 +- TimberWinR/Inputs/InputListener.cs | 31 ++- TimberWinR/Inputs/TailFileInputListener.cs | 13 +- TimberWinR/Inputs/TcpInputListener.cs | 18 +- TimberWinR/Inputs/WindowsEvtInputListener.cs | 15 +- TimberWinR/Manager.cs | 51 ++++- TimberWinR/Parser.cs | 203 +++++++++++++++++-- TimberWinR/TimberWinR.csproj | 4 + 17 files changed, 623 insertions(+), 235 deletions(-) create mode 100644 TimberWinR/Filters/EvaluatingFilter.cs diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs index a39ee35..ae8f015 100644 --- a/TimberWinR.ServiceHost/Program.cs +++ b/TimberWinR.ServiceHost/Program.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using TimberWinR.Outputs; @@ -22,7 +23,11 @@ namespace TimberWinR.ServiceHost { Arguments arguments = new Arguments(); + var text = "Nov 21 17:27:53"; + var pattern = "MMM dd HH:mm:ss"; + var match = Regex.Match(text, pattern); + Type x = Type.GetType("string"); Type x1 = Type.GetType("System.string"); @@ -95,8 +100,9 @@ namespace TimberWinR.ServiceHost /// private void RunService() { - TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile); + TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken); +#if false var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken); _nlogListener = new TcpInputListener(_cancellationToken, 5140); @@ -119,6 +125,7 @@ namespace TimberWinR.ServiceHost var elistner = new TailFileInputListener(logConfig, _cancellationToken); outputRedis.Connect(elistner); } +#endif } } } diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json index cade4cd..0e2d4cf 100644 --- a/TimberWinR.ServiceHost/config.json +++ b/TimberWinR.ServiceHost/config.json @@ -1,76 +1,112 @@ -{ - "TimberWinR":{ - "Inputs":{ - "WindowsEvents":[ - { - "source":"System,Application", - "binaryFormat":"PRINT", - "resolveSIDS":true +{ + "TimberWinR": { + "Inputs": { + "WindowsEvents": [ + { + "source": "System,Application", + "binaryFormat": "PRINT", + "resolveSIDS": true + } + ], + "Tcp": [ + { + "port": "5140" + } + ], + "Logs": [ + { + "name": "Syslogs1", + "location": "C:\\Logs1\\*.log" + } + ], + "IISW3CLogs": [ + { + "name": "Default site", + "location": "c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*" + } + ] + }, + "Outputs": { + "Redis": [ + { + "host": [ + "logaggregator.vistaprint.svc" + ] + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "[type] == \"Win32-FileLog\"", + "match": [ + "Text", + "" + ], + "add_field": [ + "host", + "%{ComputerName}" + ] + } + }, + { + "grok": { + "condition": "[type] == \"Win32-Eventlog\"", + "match": [ + "Message", + "" + ], + "remove_field": [ + "ComputerName" + ] + } + }, + { + "grok": { + "match": [ + "message", + "%{SYSLOGLINE}" + ], + "add_tag": [ + "rn_%{Index}", + "bar" + ], + "add_field": [ + "foo_%{logsource}", + "Hello dude from %{ComputerName}" + ] + } + }, + { + "grok": { + "match": [ + "Text", + "%{SYSLOGLINE}" + ], + "add_tag": [ + "rn_%{RecordNumber}", + "bar" + ] + } + }, + { + "mutate": { + "rename": [ + "host", "Host", + "message","Message", + "SID", "Username" + ] + } + }, + { + "date": { + "match": [ + "timestamp", + "MMM d HH:mm:sss", + "MMM dd HH:mm:ss" + ] + } } - ], - "Logs":[ - { - "name":"Syslogs1", - "location":"C:\\Logs1\\*.log" - } - ], - "IISW3CLogs":[ - { - "name":"Default site", - "location":"c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*" - } - ] - }, - "Filters":[ - { - "grok":{ - "condition": "[type] == \"Win32-FileLog\"", - "match":[ - "Text", - "" - ], - "add_field":[ - "host", - "%{ComputerName}" - ] - } - }, - { - "grok":{ - "match":[ - "message", - "%{SYSLOGLINE}" - ], - "add_tag":[ - "rn_%{Index}", - "bar" - ], - "add_field":[ - "foo_%{logsource}", - "Hello dude from %{ComputerName}" - ] - } - }, - { - "grok":{ - "match":[ - "Text", - "%{SYSLOGLINE}" - ], - "add_tag":[ - "rn_%{RecordNumber}", - "bar" - ] - } - }, - { - "mutate":{ - "rename":[ - "message", - "Message" - ] - } - } - ] - } -} \ No newline at end of file + ] + } +} diff --git a/TimberWinR.UnitTests/Configuration.cs b/TimberWinR.UnitTests/Configuration.cs index 11430df..9a12709 100644 --- a/TimberWinR.UnitTests/Configuration.cs +++ b/TimberWinR.UnitTests/Configuration.cs @@ -7,13 +7,69 @@ using System.Threading.Tasks; using TimberWinR; using TimberWinR.Inputs; using TimberWinR.Filters; +using Newtonsoft.Json.Linq; namespace TimberWinR.UnitTests { [TestFixture] public class ConfigurationTest { - - + [Test] + public void TestInvalidMatchConfig() + { + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"" + ] + } + }] + } + }"; + + try + { + Configuration c = Configuration.FromString(grokJson); + Assert.IsTrue(false, "Should have thrown an exception"); + } + catch (TimberWinR.Parser.Grok.GrokFilterException ex) + { + } + } + + [Test] + public void TestInvalidAddTagConfig() + { + string grokJson = @"{ + ""TimberWinR"":{ + ""Filters"":[ + { + ""grok"":{ + ""condition"": ""[type] == \""Win32-FileLog\"""", + ""match"":[ + ""Text"", """" + ], + ""add_tag"": [ + ""rn_%{Index}"", + ], + } + }] + } + }"; + + try + { + Configuration c = Configuration.FromString(grokJson); + Assert.IsTrue(false, "Should have thrown an exception"); + } + catch (TimberWinR.Parser.Grok.GrokAddTagException ex) + { + } + } + } } diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index 87fc3b7..a255fc3 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -192,7 +192,6 @@ namespace TimberWinR.UnitTests c = Configuration.FromString(grokJson3); grok = c.Filters.First() as Grok; Assert.IsFalse(grok.Apply(json)); - } [Test] diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 6a1405b..a7309fe 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data.Odbc; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Text; @@ -17,6 +18,7 @@ using TimberWinR.Filters; using NLog; using TimberWinR.Parser; +using Topshelf.Configurators; using IISW3CLog = TimberWinR.Parser.IISW3CLog; using WindowsEvent = TimberWinR.Parser.WindowsEvent; @@ -25,14 +27,25 @@ namespace TimberWinR public class Configuration { private List _events = new List(); - public IEnumerable Events { get { return _events; } } - private List _logs = new List(); + private List _redisOutputs = new List(); + public IEnumerable RedisOutputs + { + get { return _redisOutputs; } + } + + private List _tcps = new List(); + public IEnumerable Tcps + { + get { return _tcps; } + } + + private List _logs = new List(); public IEnumerable Logs { get { return _logs; } @@ -78,23 +91,47 @@ namespace TimberWinR if (x.TimberWinR.Inputs != null) { - c._events = x.TimberWinR.Inputs.WindowsEvents.ToList(); + c._events = x.TimberWinR.Inputs.WindowsEvents.ToList(); c._iisw3clogs = x.TimberWinR.Inputs.IISW3CLogs.ToList(); - c._logs = x.TimberWinR.Inputs.Logs.ToList(); + c._logs = x.TimberWinR.Inputs.Logs.ToList(); + c._redisOutputs = x.TimberWinR.Outputs.Redis.ToList(); + c._tcps = x.TimberWinR.Inputs.Tcps.ToList(); } if (x.TimberWinR.Filters != null) c._filters = x.TimberWinR.AllFilters.ToList(); - + c.Validate(c); + + // Validate return c; } + + void Validate(Configuration c) + { + try + { + foreach (var e in c.Events) + e.Validate(); + + foreach (var f in c.Filters) + f.Validate(); + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + throw ex; + } + } + public Configuration() { _filters = new List(); _events = new List(); _iisw3clogs = new List(); _logs = new List(); + _redisOutputs = new List(); + _tcps = new List(); } public static Object GetPropValue(String name, Object obj) diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs index 78db43d..139f44b 100644 --- a/TimberWinR/Filters/DateFilter.cs +++ b/TimberWinR/Filters/DateFilter.cs @@ -6,69 +6,63 @@ using System.Text; using System.Xml; using Newtonsoft.Json.Linq; using System.Xml.Linq; +using TimberWinR.Parser; +using RapidRegex.Core; +using System.Text.RegularExpressions; -namespace TimberWinR.Filters +namespace TimberWinR.Parser { - public class DateFilter : FilterBase - { - public new const string TagName = "Date"; - - public string Field { get; private set; } - public string Target { get; private set; } - public bool ConvertToUTC { get; private set; } - public List Patterns { get; private set; } - - public static void Parse(List filters, XElement dateElement) + public partial class DateFilter : LogstashFilter + { + public override bool Apply(JObject json) { - filters.Add(parseDate(dateElement)); - } - - static DateFilter parseDate(XElement e) - { - return new DateFilter(e); - } - - DateFilter(XElement parent) - { - Patterns = new List(); - - Field = ParseStringAttribute(parent, "field"); - Target = ParseStringAttribute(parent, "target", Field); - ConvertToUTC = ParseBoolAttribute(parent, "convertToUTC", false); - ParsePatterns(parent); - } - - - private void ParsePatterns(XElement parent) - { - foreach (var e in parent.Elements("Pattern")) + if (Matches(json)) { - string pattern = e.Value; - Patterns.Add(pattern); + ApplyFilter(json); + } + + return true; + } + + + private void ApplyFilter(JObject json) + { + string text = json.ToString(); + if (!string.IsNullOrEmpty(text)) + { + DateTime ts; + if (Patterns == null || Patterns.Length == 0) + { + if (DateTime.TryParse(text, out ts)) + AddOrModify(json, ts); + } + else + { + if (DateTime.TryParseExact(text, Patterns.ToArray(), CultureInfo.InvariantCulture, + DateTimeStyles.None, out ts)) + AddOrModify(json, ts); + } } } - public override void Apply(JObject json) + private bool Matches(Newtonsoft.Json.Linq.JObject json) { + string field = Match[0]; + JToken token = null; - if (json.TryGetValue(Field, StringComparison.OrdinalIgnoreCase, out token)) + if (json.TryGetValue(field, out token)) { string text = token.ToString(); if (!string.IsNullOrEmpty(text)) { DateTime ts; - if (Patterns == null || Patterns.Count == 0) - { - if (DateTime.TryParse(text, out ts)) - AddOrModify(json, ts); - } - else - { - if (DateTime.TryParseExact(text, Patterns.ToArray(), CultureInfo.InvariantCulture, DateTimeStyles.None, out ts)) - AddOrModify(json, ts); - } + var exprArray = Match.Skip(1).ToArray(); + if (DateTime.TryParseExact(text, exprArray, CultureInfo.InvariantCulture,DateTimeStyles.None, out ts)) + AddOrModify(json, ts); } + return true; // Empty field is no match } + return false; // Not specified is failure } @@ -77,10 +71,10 @@ namespace TimberWinR.Filters if (ConvertToUTC) ts = ts.ToUniversalTime(); - if (json[Target] == null) - json.Add(Target, ts); - else - json[Target] = ts; + //if (json[Target] == null) + // json.Add(Target, ts); + //else + // json[Target] = ts; } } } diff --git a/TimberWinR/Filters/EvaluatingFilter.cs b/TimberWinR/Filters/EvaluatingFilter.cs new file mode 100644 index 0000000..16a669b --- /dev/null +++ b/TimberWinR/Filters/EvaluatingFilter.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.CSharp; +using Newtonsoft.Json.Linq; +using NLog; + +namespace TimberWinR.Filters +{ + class EvaluatingFilter + { + protected bool EvaluateCondition(JObject json, string condition) + { + // Create a new instance of the C# compiler + var cond = condition; + + IList keys = json.Properties().Select(p => p.Name).ToList(); + foreach (string key in keys) + cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); + + var compiler = new CSharpCodeProvider(); + + // Create some parameters for the compiler + var parms = new System.CodeDom.Compiler.CompilerParameters + { + GenerateExecutable = false, + GenerateInMemory = true + }; + parms.ReferencedAssemblies.Add("System.dll"); + var code = string.Format(@" using System; + class EvaluatorClass + {{ + public bool Evaluate() + {{ + return {0}; + }} + }}", cond); + + // Try to compile the string into an assembly + var results = compiler.CompileAssemblyFromSource(parms, new string[] { code }); + + // If there weren't any errors get an instance of "MyClass" and invoke + // the "Message" method on it + if (results.Errors.Count == 0) + { + var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass"); + var result = evClass.GetType(). + GetMethod("Evaluate"). + Invoke(evClass, null); + return bool.Parse(result.ToString()); + } + else + { + foreach (var e in results.Errors) + { + LogManager.GetCurrentClassLogger().Error(e); + LogManager.GetCurrentClassLogger().Error("Bad Code: {0}", code); + } + } + + return false; + } + } +} diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index 128e9aa..d286ec6 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -34,7 +34,7 @@ namespace TimberWinR.Parser { public override bool Apply(JObject json) { - if (Condition != null && !EvaluateCondition(json)) + if (Condition != null && !EvaluateCondition(json, Condition)) return false; if (Matches(json)) @@ -48,55 +48,6 @@ namespace TimberWinR.Parser return false; } - private bool EvaluateCondition(JObject json) - { - // Create a new instance of the C# compiler - var cond = Condition; - - IList keys = json.Properties().Select(p => p.Name).ToList(); - foreach (string key in keys) - cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); - - var compiler = new CSharpCodeProvider(); - - // Create some parameters for the compiler - var parms = new System.CodeDom.Compiler.CompilerParameters - { - GenerateExecutable = false, - GenerateInMemory = true - }; - parms.ReferencedAssemblies.Add("System.dll"); - var code = string.Format(@" using System; - class EvaluatorClass - {{ - public bool Evaluate() - {{ - return {0}; - }} - }}", cond); - - // Try to compile the string into an assembly - var results = compiler.CompileAssemblyFromSource(parms, new string[] { code }); - - // If there weren't any errors get an instance of "MyClass" and invoke - // the "Message" method on it - if (results.Errors.Count == 0) - { - var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass"); - var result = evClass.GetType(). - GetMethod("Evaluate"). - Invoke(evClass, null); - return bool.Parse(result.ToString()); - } - else - { - foreach (var e in results.Errors) - LogManager.GetCurrentClassLogger().Error(e); - } - - return false; - } - private bool Matches(Newtonsoft.Json.Linq.JObject json) { string field = Match[0]; diff --git a/TimberWinR/Filters/MutateFilter.cs b/TimberWinR/Filters/MutateFilter.cs index 55f71f9..90865b4 100644 --- a/TimberWinR/Filters/MutateFilter.cs +++ b/TimberWinR/Filters/MutateFilter.cs @@ -13,6 +13,9 @@ namespace TimberWinR.Parser { public override bool Apply(JObject json) { + if (Condition != null && !EvaluateCondition(json, Condition)) + return false; + ApplySplits(json); ApplyRenames(json); ApplyReplace(json); diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index c2d8c27..a25128e 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -25,7 +25,7 @@ namespace TimberWinR.Inputs public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) - : base(cancelToken) + : base(cancelToken, "Win32-IISLog") { _arguments = arguments; _pollingIntervalInSeconds = pollingIntervalInSeconds; @@ -92,8 +92,7 @@ namespace TimberWinR.Inputs } else json.Add(new JProperty(field.Name, v)); - } - json.Add(new JProperty("type", "Win32-IISLog")); + } ProcessJson(json); } } diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs index e0ef091..cbec092 100644 --- a/TimberWinR/Inputs/InputListener.cs +++ b/TimberWinR/Inputs/InputListener.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json.Linq; +using System.Runtime.InteropServices; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; @@ -11,18 +12,36 @@ namespace TimberWinR.Inputs { public CancellationToken CancelToken { get; set; } public event Action OnMessageRecieved; - + private string _computerName; + private string _typeName; - public InputListener(CancellationToken token) + public InputListener(CancellationToken token, string typeName) { - this.CancelToken = token; + this.CancelToken = token; + this._typeName = typeName; + this._computerName = System.Environment.MachineName + "." + + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( + @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") + .GetValue("Domain", "") + .ToString(); } + private void AddDefaultFileds(JObject json) + { + if (json["type"] == null) + json.Add(new JProperty("type", _typeName)); + + if (json["host"] == null) + json.Add(new JProperty("host", _computerName)); + } protected void ProcessJson(JObject json) - { + { if (OnMessageRecieved != null) - OnMessageRecieved(json); + { + AddDefaultFileds(json); + OnMessageRecieved(json); + } } } } diff --git a/TimberWinR/Inputs/TailFileInputListener.cs b/TimberWinR/Inputs/TailFileInputListener.cs index 6780ef8..ca58266 100644 --- a/TimberWinR/Inputs/TailFileInputListener.cs +++ b/TimberWinR/Inputs/TailFileInputListener.cs @@ -26,7 +26,7 @@ namespace TimberWinR.Inputs private TimberWinR.Parser.Log _arguments; public TailFileInputListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) - : base(cancelToken) + : base(cancelToken, "Win32-FileLog") { _arguments = arguments; _pollingIntervalInSeconds = pollingIntervalInSeconds; @@ -49,12 +49,7 @@ namespace TimberWinR.Inputs // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Location); - - string computerName = System.Environment.MachineName + "." + - Microsoft.Win32.Registry.LocalMachine.OpenSubKey( - @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") - .GetValue("Domain", "") - .ToString(); + var firstQuery = true; // Execute the query @@ -92,9 +87,7 @@ namespace TimberWinR.Inputs } else json.Add(new JProperty(field.Name, v)); - } - json.Add(new JProperty("type", "Win32-FileLog")); - json.Add(new JProperty("ComputerName", computerName)); + } ProcessJson(json); } } diff --git a/TimberWinR/Inputs/TcpInputListener.cs b/TimberWinR/Inputs/TcpInputListener.cs index cc1ce09..a83b16f 100644 --- a/TimberWinR/Inputs/TcpInputListener.cs +++ b/TimberWinR/Inputs/TcpInputListener.cs @@ -15,9 +15,12 @@ namespace TimberWinR.Inputs private readonly System.Net.Sockets.TcpListener _tcpListener; private Thread _listenThread; const int bufferSize = 16535; + private int _port; - public TcpInputListener(CancellationToken cancelToken, int port = 5140) : base(cancelToken) - { + public TcpInputListener(CancellationToken cancelToken, int port = 5140) + : base(cancelToken, "Win32-Tcp") + { + _port = port; _tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port); _listenThread = new Thread(new ThreadStart(ListenForClients)); _listenThread.Start(); @@ -32,6 +35,8 @@ namespace TimberWinR.Inputs { this._tcpListener.Start(); + LogManager.GetCurrentClassLogger().Info("Tcp Input on Port {0} Ready", _port); + while (!CancelToken.IsCancellationRequested) { try @@ -58,6 +63,12 @@ namespace TimberWinR.Inputs var tcpClient = (TcpClient)client; NetworkStream clientStream = tcpClient.GetStream(); + string computerName = System.Environment.MachineName + "." + + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( + @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") + .GetValue("Domain", "") + .ToString(); + var message = new byte[bufferSize]; while (!CancelToken.IsCancellationRequested) { @@ -83,7 +94,8 @@ namespace TimberWinR.Inputs var encoder = new ASCIIEncoding(); var encodedMessage = encoder.GetString(message, 0, bytesRead); - ProcessJson(JObject.Parse(encodedMessage)); + JObject json = JObject.Parse(encodedMessage); + ProcessJson(json); } tcpClient.Close(); } diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index 37fc249..46ad52d 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -26,7 +26,7 @@ namespace TimberWinR.Inputs private TimberWinR.Parser.WindowsEvent _arguments; public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) - : base(cancelToken) + : base(cancelToken, "Win32-Eventlog") { _arguments = arguments; _pollingIntervalInSeconds = pollingIntervalInSeconds; @@ -55,6 +55,13 @@ namespace TimberWinR.Inputs iCheckpoint = checkpointFileName, }; + string computerName = System.Environment.MachineName + "." + + Microsoft.Win32.Registry.LocalMachine.OpenSubKey( + @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") + .GetValue("Domain", "") + .ToString(); + + // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Source); @@ -76,13 +83,9 @@ namespace TimberWinR.Inputs foreach (var field in _arguments.Fields) { object v = record.getValue(field.Name); - - //if (field.FieldType == typeof(DateTime)) - // v = field.ToDateTime(v).ToUniversalTime(); - json.Add(new JProperty(field.Name, v)); } - json.Add(new JProperty("type", "Win32-Eventlog")); + ProcessJson(json); } } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index 577bd07..142992c 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -6,6 +6,9 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using TimberWinR.Inputs; +using TimberWinR.Outputs; +using System.Threading; namespace TimberWinR { @@ -15,9 +18,12 @@ namespace TimberWinR public class Manager { public Configuration Config { get; set; } - - public Manager(string xmlConfigFile, string jsonConfigFile) - { + public List Outputs { get; set; } + + public Manager(string xmlConfigFile, string jsonConfigFile, CancellationToken cancelToken) + { + Outputs = new List(); + var loggingConfiguration = new LoggingConfiguration(); // Create our default targets @@ -35,11 +41,46 @@ namespace TimberWinR LogManager.EnableLogging(); LogManager.GetCurrentClassLogger().Info("Initialized"); - - // Read the Configuration file Config = Configuration.FromFile(jsonConfigFile); + + if (Config.RedisOutputs != null) + { + foreach (var ro in Config.RedisOutputs) + { + var redis = new RedisOutput(this, ro.Host, cancelToken, ro.Index, ro.Port, ro.Timeout); + Outputs.Add(redis); + } + } + + foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C) + { + var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken); + foreach(var output in Outputs) + output.Connect(elistner); + } + + foreach (Parser.WindowsEvent eventConfig in Config.Events) + { + var elistner = new WindowsEvtInputListener(eventConfig, cancelToken); + foreach (var output in Outputs) + output.Connect(elistner); + } + + foreach (var logConfig in Config.Logs) + { + var elistner = new TailFileInputListener(logConfig, cancelToken); + foreach (var output in Outputs) + output.Connect(elistner); + } + + foreach (var tcp in Config.Tcps) + { + var elistner = new TcpInputListener(cancelToken, tcp.Port); + foreach (var output in Outputs) + output.Connect(elistner); + } } /// diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index d58a887..d869348 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -2,18 +2,28 @@ using System.Collections.Generic; using System.Configuration; using System.Linq; +using System.Reflection; +using System.Runtime.Remoting.Channels; using System.Text; +using Microsoft.CSharp; using Microsoft.SqlServer.Server; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; +using TimberWinR.Outputs; + namespace TimberWinR.Parser { - public abstract class LogstashFilter + interface IValidateSchema + { + void Validate(); + } + + public abstract class LogstashFilter : IValidateSchema { public abstract bool Apply(JObject json); - + protected void RenameProperty(JObject json, string oldName, string newName) { JToken token = json[oldName]; @@ -24,6 +34,57 @@ namespace TimberWinR.Parser } } + protected bool EvaluateCondition(JObject json, string condition) + { + // Create a new instance of the C# compiler + var cond = condition; + + IList keys = json.Properties().Select(p => p.Name).ToList(); + foreach (string key in keys) + cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); + + var compiler = new CSharpCodeProvider(); + + // Create some parameters for the compiler + var parms = new System.CodeDom.Compiler.CompilerParameters + { + GenerateExecutable = false, + GenerateInMemory = true + }; + parms.ReferencedAssemblies.Add("System.dll"); + var code = string.Format(@" using System; + class EvaluatorClass + {{ + public bool Evaluate() + {{ + return {0}; + }} + }}", cond); + + // Try to compile the string into an assembly + var results = compiler.CompileAssemblyFromSource(parms, new string[] { code }); + + // If there weren't any errors get an instance of "MyClass" and invoke + // the "Message" method on it + if (results.Errors.Count == 0) + { + var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass"); + var result = evClass.GetType(). + GetMethod("Evaluate"). + Invoke(evClass, null); + return bool.Parse(result.ToString()); + } + else + { + foreach (var e in results.Errors) + { + LogManager.GetCurrentClassLogger().Error(e); + LogManager.GetCurrentClassLogger().Error("Bad Code: {0}", code); + } + } + + return false; + } protected void RemoveProperties(JToken token, string[] fields) { JContainer container = token as JContainer; @@ -46,14 +107,12 @@ namespace TimberWinR.Parser } } - protected void ReplaceProperty(JObject json, string propertyName, string propertyValue) { if (json[propertyName] != null) json[propertyName] = propertyValue; } - protected void AddOrModify(JObject json, string fieldName, string fieldValue) { if (json[fieldName] == null) @@ -71,6 +130,9 @@ namespace TimberWinR.Parser } return fieldName; } + + public abstract void Validate(); + } [JsonObject(MemberSerialization.OptIn)] @@ -120,7 +182,7 @@ namespace TimberWinR.Parser } } - public class WindowsEvent + public class WindowsEvent : IValidateSchema { public enum FormatKinds { @@ -183,9 +245,14 @@ namespace TimberWinR.Parser Fields.Add(new Field("Message", "string")); Fields.Add(new Field("Data", "string")); } + + public void Validate() + { + + } } - public class Log + public class Log : IValidateSchema { [JsonProperty(PropertyName = "location")] public string Location { get; set; } @@ -206,9 +273,30 @@ namespace TimberWinR.Parser Fields.Add(new Field("Index", "integer")); Fields.Add(new Field("Text", "string")); } + + public void Validate() + { + + } + } + + public class Tcp : IValidateSchema + { + [JsonProperty(PropertyName = "port")] + public int Port { get; set; } + + public Tcp() + { + Port = 5140; + } + + public void Validate() + { + + } } - public class IISW3CLog + public class IISW3CLog : IValidateSchema { [JsonProperty(PropertyName = "name")] public string Name { get; set; } @@ -269,6 +357,37 @@ namespace TimberWinR.Parser Fields.Add(new Field("s-active-procs", "integer" )); Fields.Add(new Field("s-stopped-procs", "integer")); } + + public void Validate() + { + + } + } + + public partial class RedisOutput + { + [JsonProperty(PropertyName = "host")] + public string[] Host { get; set; } + [JsonProperty(PropertyName = "index")] + public string Index { get; set; } + [JsonProperty(PropertyName = "port")] + public int Port { get; set; } + [JsonProperty(PropertyName = "timeout")] + public int Timeout { get; set; } + + public RedisOutput() + { + Port = 6379; + Index = "logstash"; + Host = new string[] {"localhost"}; + Timeout = 10000; + } + } + + public class OutputTargets + { + [JsonProperty("Redis")] + public RedisOutput[] Redis { get; set; } } public class InputSources @@ -279,12 +398,30 @@ namespace TimberWinR.Parser [JsonProperty("Logs")] public Log[] Logs { get; set; } + [JsonProperty("Tcp")] + public Tcp[] Tcps { get; set; } + [JsonProperty("IISW3CLogs")] public IISW3CLog[] IISW3CLogs { get; set; } } - public partial class Grok : LogstashFilter + public partial class Grok : LogstashFilter, IValidateSchema { + public class GrokFilterException : Exception + { + public GrokFilterException() + : base("Grok filter missing required match, must be 2 array entries.") + { + } + } + + public class GrokAddTagException : Exception + { + public GrokAddTagException() + : base("Grok filter add_tag requires tuples") + { + } + } [JsonProperty("condition")] public string Condition { get; set; } @@ -304,24 +441,44 @@ namespace TimberWinR.Parser public string[] RemoveField { get; set; } [JsonProperty("remove_tag")] - public string[] RemoveTag { get; set; } + public string[] RemoveTag { get; set; } + + public override void Validate() + { + if (Match == null || Match.Length != 2) + throw new GrokFilterException(); + + if (AddTag != null && AddTag.Length%2 != 0) + throw new GrokAddTagException(); + } } - public class Date : LogstashFilter + public partial class DateFilter : LogstashFilter { - public string field { get; set; } - public string target { get; set; } - public bool convertToUTC { get; set; } - public List Pattern { get; set; } + [JsonProperty("match")] + public string[] Match { get; set; } - public override bool Apply(JObject json) + [JsonProperty("target")] + public string[] Target { get; set; } + + [JsonProperty("convertToUTC")] + public bool ConvertToUTC { get; set; } + + [JsonProperty("pattern")] + public string[] Patterns { get; set; } + + public override void Validate() { - return false; + } + } public partial class Mutate : LogstashFilter { + [JsonProperty("condition")] + public string Condition { get; set; } + [JsonProperty("rename")] public string[] Rename { get; set; } @@ -329,7 +486,12 @@ namespace TimberWinR.Parser public string[] Replace { get; set; } [JsonProperty("split")] - public string[] Split { get; set; } + public string[] Split { get; set; } + + public override void Validate() + { + + } } public class Filter @@ -339,13 +501,20 @@ namespace TimberWinR.Parser [JsonProperty("mutate")] public Mutate Mutate { get; set; } + + [JsonProperty("date")] + public DateFilter Date { get; set; } } public class TimberWinR { [JsonProperty("Inputs")] public InputSources Inputs { get; set; } + [JsonProperty("Filters")] public List Filters { get; set; } + [JsonProperty("Outputs")] + public OutputTargets Outputs { get; set; } + public LogstashFilter[] AllFilters { get diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 9da6995..b769bc3 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -66,6 +66,7 @@ + @@ -105,6 +106,9 @@ Resources.Designer.cs + + +