diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs index 0e6a47d..a39ee35 100644 --- a/TimberWinR.ServiceHost/Program.cs +++ b/TimberWinR.ServiceHost/Program.cs @@ -20,7 +20,11 @@ namespace TimberWinR.ServiceHost { private static void Main(string[] args) { - Arguments arguments = new Arguments(); + Arguments arguments = new Arguments(); + + + Type x = Type.GetType("string"); + Type x1 = Type.GetType("System.string"); HostFactory.Run(hostConfigurator => { @@ -34,6 +38,7 @@ namespace TimberWinR.ServiceHost }); hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c); + hostConfigurator.AddCommandLineDefinition("jsonFile", c => arguments.JsonFile = c); hostConfigurator.ApplyCommandLine(); hostConfigurator.RunAsLocalSystem(); @@ -49,6 +54,7 @@ namespace TimberWinR.ServiceHost internal class Arguments { public string ConfigFile { get; set; } + public string JsonFile { get; set; } public Arguments() { @@ -89,20 +95,20 @@ namespace TimberWinR.ServiceHost /// private void RunService() { - TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile); + TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile); var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken); _nlogListener = new TcpInputListener(_cancellationToken, 5140); outputRedis.Connect(_nlogListener); - foreach (Inputs.IISW3CLog iisw3cConfig in manager.Config.IISW3C) + foreach (Parser.IISW3CLog iisw3cConfig in manager.Config.IISW3C) { var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken); outputRedis.Connect(elistner); } - foreach (Inputs.WindowsEvent eventConfig in manager.Config.Events) + foreach (Parser.WindowsEvent eventConfig in manager.Config.Events) { var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken); outputRedis.Connect(elistner); diff --git a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj index a26d36e..97327eb 100644 --- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj +++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj @@ -52,6 +52,9 @@ + + PreserveNewest + diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json new file mode 100644 index 0000000..cade4cd --- /dev/null +++ b/TimberWinR.ServiceHost/config.json @@ -0,0 +1,76 @@ +{ + "TimberWinR":{ + "Inputs":{ + "WindowsEvents":[ + { + "source":"System,Application", + "binaryFormat":"PRINT", + "resolveSIDS":true + } + ], + "Logs":[ + { + "name":"Syslogs1", + "location":"C:\\Logs1\\*.log" + } + ], + "IISW3CLogs":[ + { + "name":"Default site", + "location":"c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*" + } + ] + }, + "Filters":[ + { + "grok":{ + "condition": "[type] == \"Win32-FileLog\"", + "match":[ + "Text", + "" + ], + "add_field":[ + "host", + "%{ComputerName}" + ] + } + }, + { + "grok":{ + "match":[ + "message", + "%{SYSLOGLINE}" + ], + "add_tag":[ + "rn_%{Index}", + "bar" + ], + "add_field":[ + "foo_%{logsource}", + "Hello dude from %{ComputerName}" + ] + } + }, + { + "grok":{ + "match":[ + "Text", + "%{SYSLOGLINE}" + ], + "add_tag":[ + "rn_%{RecordNumber}", + "bar" + ] + } + }, + { + "mutate":{ + "rename":[ + "message", + "Message" + ] + } + } + ] + } +} \ No newline at end of file diff --git a/TimberWinR.ServiceHost/sampleconf.xml b/TimberWinR.ServiceHost/sampleconf.xml index 33f0f16..44a296a 100644 --- a/TimberWinR.ServiceHost/sampleconf.xml +++ b/TimberWinR.ServiceHost/sampleconf.xml @@ -19,6 +19,7 @@ rn_%{RecordNumber} bar + @@ -26,6 +27,6 @@ MMM d HH:mm:ss MMM dd HH:mm:ss ISO8601 - + diff --git a/TimberWinR.UnitTests/Configuration.cs b/TimberWinR.UnitTests/Configuration.cs index 9b18ca7..11430df 100644 --- a/TimberWinR.UnitTests/Configuration.cs +++ b/TimberWinR.UnitTests/Configuration.cs @@ -13,298 +13,7 @@ namespace TimberWinR.UnitTests [TestFixture] public class ConfigurationTest { - Configuration c = new Configuration("testconf.xml"); - - public void OutputEvents() - { - foreach (var evt in c.Events) - Console.WriteLine(evt); - } - - public void OutputLogs() - { - foreach (var log in c.Logs) - Console.WriteLine(log); - } - - public void OutputIIS() - { - foreach (var iis in c.IIS) - Console.WriteLine(iis); - } - - public void OutputIISW3C() - { - foreach (var iisw3c in c.IISW3C) - Console.WriteLine(iisw3c); - } - - public void OutputFilters() - { - foreach (var filter in c.Filters) - Console.WriteLine(filter); - } - - [Test] - public void Output() - { - OutputEvents(); - OutputLogs(); - OutputIIS(); - OutputIISW3C(); - OutputFilters(); - } - - [Test] - public void NumOfEvents() - { - Assert.AreEqual(1, c.Events.ToArray().Length); - } - - [Test] - public void NumOfLogs() - { - Assert.AreEqual(3, c.Logs.ToArray().Length); - } - - [Test] - public void NumOfIIS() - { - Assert.AreEqual(0, c.IIS.ToArray().Length); - } - - [Test] - public void NumOfIISW3C() - { - Assert.AreEqual(1, c.IISW3C.ToArray().Length); - } - - [Test] - public void FieldsOfEvents() - { - Dictionary fields = new Dictionary() - { - { "EventLog", typeof(string) }, - { "RecordNumber", typeof(int) }, - { "TimeGenerated", typeof(DateTime) }, - { "TimeWritten", typeof(DateTime) }, - { "EventID", typeof(int) }, - { "EventType", typeof(int) }, - { "EventTypeName", typeof(string) }, - { "EventCategory", typeof(int) }, - { "EventCategoryName", typeof(string) }, - { "SourceName", typeof(string) }, - { "Strings", typeof(string) }, - { "ComputerName", typeof(string) }, - { "SID", typeof(string) }, - { "Message", typeof(string) }, - { "Data", typeof(string) } - }; - foreach (FieldDefinition field in c.Events.ToArray()[0].Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - } - - [Test] - public void FieldsOfLogs() - { - Dictionary fields = new Dictionary() - { - { "LogFilename", typeof(string) }, - { "Index", typeof(int) }, - { "Text", typeof(string) } - }; - foreach (FieldDefinition field in c.Logs.ToArray()[0].Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - foreach (FieldDefinition field in c.Logs.ToArray()[1].Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - foreach (FieldDefinition field in c.Logs.ToArray()[2].Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - - } - - [Test] - public void FieldsOfIIS() - { - Dictionary fields = new Dictionary() - { - { "LogFilename", typeof(string) }, - { "LogRow", typeof(int) }, - { "UserIP", typeof(string) }, - { "UserName", typeof(string) }, - { "Date", typeof(DateTime) }, - { "Time", typeof(DateTime) }, - { "ServiceInstance", typeof(string) }, - { "HostName", typeof(string) }, - { "ServerIP", typeof(string) }, - { "TimeTaken", typeof(int) }, - { "BytesSent", typeof(int) }, - { "BytesReceived", typeof(int) }, - { "StatusCode", typeof(int) }, - { "Win32StatusCode", typeof(int) }, - { "RequestType", typeof(string) }, - { "Target", typeof(string) }, - { "Parameters", typeof(string) } - }; - - foreach (var iis in c.IIS.ToArray()) - { - foreach (FieldDefinition field in iis.Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - } - - } - - [Test] - public void FieldsOfIISW3C() - { - Dictionary fields = new Dictionary() - { - { "LogFilename", typeof(string) }, - { "LogRow", typeof(int) }, - { "date", typeof(DateTime) }, - { "time", typeof(DateTime) }, - { "c-ip", typeof(string) }, - { "cs-username", typeof(string) }, - { "s-sitename", typeof(string) }, - { "s-computername", typeof(int) }, - { "s-ip", typeof(string) }, - { "s-port", typeof(int) }, - { "cs-method", typeof(string) }, - { "cs-uri-stem", typeof(string) }, - { "cs-uri-query", typeof(string) }, - { "sc-status", typeof(int) }, - { "sc-substatus", typeof(int) }, - { "sc-win32-status", typeof(int) }, - { "sc-bytes", typeof(int) }, - { "cs-bytes", typeof(int) }, - { "time-taken", typeof(int) }, - { "cs-version", typeof(string) }, - { "cs-host", typeof(string) }, - { "cs(User-Agent)", typeof(string) }, - { "cs(Cookie)", typeof(string) }, - { "cs(Referer)", typeof(string) }, - { "s-event", typeof(string) }, - { "s-process-type", typeof(string) }, - { "s-user-time", typeof(double) }, - { "s-kernel-time", typeof(double) }, - { "s-page-faults", typeof(int) }, - { "s-total-procs", typeof(int) }, - { "s-active-procs", typeof(int) }, - { "s-stopped-procs", typeof(int) } - }; - foreach (FieldDefinition field in c.IISW3C.ToArray()[0].Fields) - { - Assert.Contains(field.Name, fields.Keys); - } - } - - [Test] - public void ParametersOfEvents() - { - string source = "System,Application"; - bool fullText = true; - bool resolveSIDS = true; - bool formatMsg = true; - string msgErrorMode = "MSG"; - bool fullEventCode = false; - string direction = "FW"; - string stringsSep = "|"; - string binaryFormat = "PRINT"; - - TimberWinR.Inputs.WindowsEvent evt = c.Events.ToArray()[0]; - - Assert.AreEqual(source, evt.Source); - Assert.AreEqual(fullText, evt.FullText); - Assert.AreEqual(resolveSIDS, evt.ResolveSIDS); - Assert.AreEqual(formatMsg, evt.FormatMsg); - Assert.AreEqual(msgErrorMode, evt.MsgErrorMode); - Assert.AreEqual(fullEventCode, evt.FullEventCode); - Assert.AreEqual(direction, evt.Direction); - Assert.AreEqual(stringsSep, evt.StringsSep); - Assert.IsNull(evt.ICheckpoint); - Assert.AreEqual(binaryFormat, evt.BinaryFormat); - } - - [Test] - public void ParametersOfLogs() - { - string name = "First Set"; - string location = @"C:\Logs1\*.log"; - int iCodepage = 0; - int recurse = 0; - bool splitLongLines = false; - - TimberWinR.Inputs.TailFileInput log = c.Logs.ToArray()[0]; - - Assert.AreEqual(name, log.Name); - Assert.AreEqual(location, log.Location); - Assert.AreEqual(iCodepage, log.ICodepage); - Assert.AreEqual(recurse, log.Recurse); - Assert.AreEqual(splitLongLines, log.SplitLongLines); - - - name = "Second Set"; - location = @"C:\Logs2\*.log"; - iCodepage = 0; - recurse = 0; - splitLongLines = false; - - log = c.Logs.ToArray()[1]; - - Assert.AreEqual(name, log.Name); - Assert.AreEqual(location, log.Location); - Assert.AreEqual(iCodepage, log.ICodepage); - Assert.AreEqual(recurse, log.Recurse); - Assert.AreEqual(splitLongLines, log.SplitLongLines); - - - name = "Third Set"; - location = @"C:\Logs2\1.log,C:\Logs2\2.log"; - iCodepage = 0; - recurse = 0; - splitLongLines = false; - - log = c.Logs.ToArray()[2]; - - Assert.AreEqual(name, log.Name); - Assert.AreEqual(location, log.Location); - Assert.AreEqual(iCodepage, log.ICodepage); - Assert.AreEqual(recurse, log.Recurse); - Assert.AreEqual(splitLongLines, log.SplitLongLines); - } - - [Test] - public void ParametersOfIISW3C() - { - string name = "Default site"; - string location = @"c:\inetpub\logs\LogFiles\W3SVC1\*"; - int iCodepage = -2; - int recurse = 0; - bool dQuotes = false; - bool dirTime = false; - bool consolidateLogs = false; - - TimberWinR.Inputs.IISW3CLog iisw3c = c.IISW3C.ToArray()[0]; - - Assert.AreEqual(name, iisw3c.Name); - Assert.AreEqual(location, iisw3c.Location); - Assert.AreEqual(iCodepage, iisw3c.ICodepage); - Assert.AreEqual(recurse, iisw3c.Recurse); - Assert.IsNull(iisw3c.MinDateMod); - Assert.AreEqual(dQuotes, iisw3c.DQuotes); - Assert.AreEqual(dirTime, iisw3c.DirTime); - Assert.AreEqual(consolidateLogs, iisw3c.ConsolidateLogs); - Assert.IsEmpty(iisw3c.ICheckpoint); - } + + } } diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 42037f5..6a1405b 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -2,162 +2,114 @@ using System.Collections.Generic; using System.Data.Odbc; using System.Linq; +using System.Reflection; using System.Text; using System.Xml; using System.Xml.Linq; using System.IO; using System.Globalization; using System.Xml.Schema; - +using Newtonsoft.Json; +using Newtonsoft.Json.Bson; +using Newtonsoft.Json.Linq; using TimberWinR.Inputs; using TimberWinR.Filters; using NLog; +using TimberWinR.Parser; +using IISW3CLog = TimberWinR.Parser.IISW3CLog; +using WindowsEvent = TimberWinR.Parser.WindowsEvent; namespace TimberWinR { public class Configuration { - - private static List _events = new List(); + private List _events = new List(); public IEnumerable Events { get { return _events; } } - private static List _logs = new List(); + private List _logs = new List(); - public IEnumerable Logs + public IEnumerable Logs { get { return _logs; } - } + } - private static List _iislogs = new List(); - - public IEnumerable IIS - { - get { return _iislogs; } - } - - private static List _iisw3clogs = new List(); + private List _iisw3clogs = new List(); public IEnumerable IISW3C { get { return _iisw3clogs; } } - private static List _filters = new List(); + private List _filters = new List(); - public IEnumerable Filters + public IEnumerable Filters { get { return _filters; } } - public Configuration(string xmlConfFile) + public static Configuration FromFile(string jsonConfFile) { - validateWithSchema(xmlConfFile, Properties.Resources.configSchema); + Configuration c = new Configuration(); - try + if (!string.IsNullOrEmpty(jsonConfFile)) { - parseConfInput(xmlConfFile); - parseConfFilter(xmlConfFile); + string json = File.ReadAllText(jsonConfFile); + + return FromString(json); } - catch(Exception ex) + + return null; + } + + public static Configuration FromString(string json) + { + Configuration c = new Configuration(); + + JsonSerializer serializer = new JsonSerializer(); + TextReader re = new StringReader(json); + JsonTextReader reader = new JsonTextReader(re); + + var x = serializer.Deserialize(reader); + + if (x.TimberWinR.Inputs != null) { - LogManager.GetCurrentClassLogger().Error(ex); + c._events = x.TimberWinR.Inputs.WindowsEvents.ToList(); + c._iisw3clogs = x.TimberWinR.Inputs.IISW3CLogs.ToList(); + c._logs = x.TimberWinR.Inputs.Logs.ToList(); } + + if (x.TimberWinR.Filters != null) + c._filters = x.TimberWinR.AllFilters.ToList(); + + + return c; + } + public Configuration() + { + _filters = new List(); + _events = new List(); + _iisw3clogs = new List(); + _logs = new List(); } - private static void validateWithSchema(string xmlConfFile, string xsdSchema) + public static Object GetPropValue(String name, Object obj) { - XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri); - - // Ensure that the xml configuration file provided obeys the xsd schema. - XmlSchemaSet schemas = new XmlSchemaSet(); - schemas.Add("", XmlReader.Create(new StringReader(xsdSchema))); -#if true - bool errorsFound = false; - config.Validate(schemas, (o, e) => + foreach (String part in name.Split('.')) { - errorsFound = true; - LogManager.GetCurrentClassLogger().Error(e.Message); - }, true); + if (obj == null) { return null; } - if (errorsFound) - DumpInvalidNodes(config.Root); -#endif - } + Type type = obj.GetType(); + PropertyInfo info = type.GetProperty(part); + if (info == null) { return null; } - static void DumpInvalidNodes(XElement el) - { - if (el.GetSchemaInfo().Validity != XmlSchemaValidity.Valid) - LogManager.GetCurrentClassLogger().Error("Invalid Element {0}", - el.AncestorsAndSelf() - .InDocumentOrder() - .Aggregate("", (s, i) => s + "/" + i.Name.ToString())); - foreach (XAttribute att in el.Attributes()) - if (att.GetSchemaInfo().Validity != XmlSchemaValidity.Valid) - LogManager.GetCurrentClassLogger().Error("Invalid Attribute {0}", - att - .Parent - .AncestorsAndSelf() - .InDocumentOrder() - .Aggregate("", - (s, i) => s + "/" + i.Name.ToString()) + "/@" + att.Name.ToString() - ); - foreach (XElement child in el.Elements()) - DumpInvalidNodes(child); - } - - static void parseConfInput(string xmlConfFile) - { - XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri); - - XElement allInputs = config.Root.Element(InputBase.TagName); - if (allInputs == null) - throw new TimberWinR.ConfigurationErrors.MissingRequiredTagException(InputBase.TagName); - - createInputs(allInputs, WindowsEvent.ParentTagName, WindowsEvent.TagName, _events, WindowsEvent.Parse); - createInputs(allInputs, TailFileInput.ParentTagName, TailFileInput.TagName, _logs, TailFileInput.Parse); - createInputs(allInputs, IISLog.ParentTagName, IISLog.TagName, _iislogs, IISLog.Parse); - createInputs(allInputs, IISW3CLog.ParentTagName, IISW3CLog.TagName, _iisw3clogs, IISW3CLog.Parse); - } - - static void createInputs(XElement allInputs, string parentTagName, string tagName, List inputList, Action, XElement> parse) - { - IEnumerable inputs = - from el in allInputs.Elements(parentTagName).Elements(tagName) - select el; - foreach (XElement input in inputs) - parse(inputList, input); - } - - static void parseConfFilter(string xmlConfFile) - { - XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri); - - IEnumerable filters = - from el in config.Root.Elements(FilterBase.TagName) - select el; - - foreach (XElement e in filters.Elements()) - { - switch (e.Name.ToString()) - { - case DateFilter.TagName: - DateFilter.Parse(_filters, e); - break; - case GrokFilter.TagName: - GrokFilter.Parse(_filters, e); - break; - case MutateFilter.TagName: - MutateFilter.Parse(_filters, e); - break; - default: - throw new Exception(string.Format("Unknown tag: {0}", e.Name.ToString())); - } + obj = info.GetValue(obj, null); } - } + return obj; + } } } \ No newline at end of file diff --git a/TimberWinR/Inputs/IISLog.cs b/TimberWinR/Inputs/IISLog.cs deleted file mode 100644 index 93f67c0..0000000 --- a/TimberWinR/Inputs/IISLog.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Text; -using System.Xml.Linq; - -namespace TimberWinR.Inputs -{ - public class IISLog : InputBase - { - public const string ParentTagName = "IISLogs"; - public new const string TagName = "IISLog"; - - public string Name { get; private set; } - public string Location { get; private set; } - public int ICodepage { get; private set; } - public int Recurse { get; private set; } - public string MinDateMod { get; private set; } - public string Locale { get; private set; } - public string ICheckpoint { get; private set; } - public List Fields { get; private set; } - - public static void Parse(List iislogs, XElement iislogElement) - { - iislogs.Add(parseIISLog(iislogElement)); - } - - static IISLog parseIISLog(XElement e) - { - return new IISLog(e); - } - - public IISLog(XElement parent) - { - Name = ParseRequiredStringAttribute(parent, "name"); - Location = ParseRequiredStringAttribute(parent, "location"); - ICodepage = ParseIntAttribute(parent, "iCodepage", -2); - Recurse = ParseIntAttribute(parent, "recurse", 0); - MinDateMod = ParseDateAttribute(parent, "minDateMod"); - Locale = ParseStringAttribute(parent, "locale", "DEF"); - ICheckpoint = ParseStringAttribute(parent, "iCheckpoint"); - ParseFields(parent); - } - - private void ParseFields(XElement parent) - { - Dictionary allPossibleFields = new Dictionary() - { - { "LogFilename", typeof(string) }, - { "LogRow", typeof(int) }, - { "UserIP", typeof(string) }, - { "UserName", typeof(string) }, - { "Date", typeof(DateTime) }, - { "Time", typeof(DateTime) }, - { "ServiceInstance", typeof(string) }, - { "HostName", typeof(string) }, - { "ServerIP", typeof(string) }, - { "TimeTaken", typeof(int) }, - { "BytesSent", typeof(int) }, - { "BytesReceived", typeof(int) }, - { "StatusCode", typeof(int) }, - { "Win32StatusCode", typeof(int) }, - { "RequestType", typeof(string) }, - { "Target", typeof(string) }, - { "Parameters", typeof(string) } - }; - - Fields = base.parseFields(parent, allPossibleFields); - } - - } -} diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index f2ba8d9..c2d8c27 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -21,10 +21,10 @@ namespace TimberWinR.Inputs public class IISW3CInputListener : InputListener { private int _pollingIntervalInSeconds = 1; - private TimberWinR.Inputs.IISW3CLog _arguments; + private TimberWinR.Parser.IISW3CLog _arguments; - public IISW3CInputListener(TimberWinR.Inputs.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) + public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken) { _arguments = arguments; @@ -42,17 +42,17 @@ namespace TimberWinR.Inputs var iFmt = new IISW3CLogInputFormat() { - codepage = _arguments.ICodepage, + codepage = _arguments.CodePage, consolidateLogs = _arguments.ConsolidateLogs, dirTime = _arguments.DirTime, - dQuotes = _arguments.DQuotes, + dQuotes = _arguments.DoubleQuotes, iCheckpoint = checkpointFileName, recurse = _arguments.Recurse, - useDoubleQuotes = _arguments.DQuotes + useDoubleQuotes = _arguments.DoubleQuotes }; - if (!string.IsNullOrEmpty(_arguments.MinDateMod)) - iFmt.minDateMod = _arguments.MinDateMod; + if (_arguments.MinDateMod.HasValue) + iFmt.minDateMod = _arguments.MinDateMod.Value.ToString("yyyy-MM-dd hh:mm:ss"); // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Location); @@ -84,12 +84,14 @@ namespace TimberWinR.Inputs if (!colMap.ContainsKey(field.Name)) continue; - object v = record.getValue(field.Name); - - if (field.FieldType == typeof(DateTime)) - v = field.ToDateTime(v).ToUniversalTime(); - - json.Add(new JProperty(field.Name, v)); + object v = record.getValue(field.Name); + if (field.DataType == typeof(DateTime)) + { + DateTime dt = DateTime.Parse(v.ToString()); + json.Add(new JProperty(field.Name, dt)); + } + else + json.Add(new JProperty(field.Name, v)); } json.Add(new JProperty("type", "Win32-IISLog")); ProcessJson(json); diff --git a/TimberWinR/Inputs/IISW3CLog.cs b/TimberWinR/Inputs/IISW3CLog.cs deleted file mode 100644 index 06cd239..0000000 --- a/TimberWinR/Inputs/IISW3CLog.cs +++ /dev/null @@ -1,89 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Xml.Linq; - -namespace TimberWinR.Inputs -{ - public class IISW3CLog : InputBase - { - public const string ParentTagName = "IISW3CLogs"; - public new const string TagName = "IISW3CLog"; - - public string Name { get; private set; } - public string Location { get; private set; } - public int ICodepage { get; private set; } - public int Recurse { get; private set; } - public string MinDateMod { get; private set; } - public bool DQuotes { get; private set; } - public bool DirTime { get; private set; } - public bool ConsolidateLogs { get; private set; } - public string ICheckpoint { get; private set; } - public List Fields { get; private set; } - - public static void Parse(List iisw3clogs, XElement iisw3clogElement) - { - iisw3clogs.Add(parseIISW3CLog(iisw3clogElement)); - } - - static IISW3CLog parseIISW3CLog(XElement e) - { - return new IISW3CLog(e); - } - - public IISW3CLog(XElement parent) - { - Name = ParseRequiredStringAttribute(parent, "name"); - Location = ParseRequiredStringAttribute(parent, "location"); - ICodepage = ParseIntAttribute(parent, "iCodepage", -2); - Recurse = ParseIntAttribute(parent, "recurse", 0); - DQuotes = ParseBoolAttribute(parent, "dQuotes", false); - DirTime = ParseBoolAttribute(parent, "dirTime", false); - ConsolidateLogs = ParseBoolAttribute(parent, "consolidateLogs", false); - ICheckpoint = ParseStringAttribute(parent, "iCheckpoint"); - ParseFields(parent); - } - - private void ParseFields(XElement parent) - { - Dictionary allPossibleFields = new Dictionary() - { - { "LogFilename", typeof(string) }, - { "LogRow", typeof(int) }, - { "date", typeof(DateTime) }, - { "time", typeof(DateTime) }, - { "c-ip", typeof(string) }, - { "cs-username", typeof(string) }, - { "s-sitename", typeof(string) }, - { "s-computername", typeof(int) }, - { "s-ip", typeof(string) }, - { "s-port", typeof(int) }, - { "cs-method", typeof(string) }, - { "cs-uri-stem", typeof(string) }, - { "cs-uri-query", typeof(string) }, - { "sc-status", typeof(int) }, - { "sc-substatus", typeof(int) }, - { "sc-win32-status", typeof(int) }, - { "sc-bytes", typeof(int) }, - { "cs-bytes", typeof(int) }, - { "time-taken", typeof(int) }, - { "cs-version", typeof(string) }, - { "cs-host", typeof(string) }, - { "cs(User-Agent)", typeof(string) }, - { "cs(Cookie)", typeof(string) }, - { "cs(Referer)", typeof(string) }, - { "s-event", typeof(string) }, - { "s-process-type", typeof(string) }, - { "s-user-time", typeof(double) }, - { "s-kernel-time", typeof(double) }, - { "s-page-faults", typeof(int) }, - { "s-total-procs", typeof(int) }, - { "s-active-procs", typeof(int) }, - { "s-stopped-procs", typeof(int) } - }; - - Fields = base.parseFields(parent, allPossibleFields); - } - - } -} diff --git a/TimberWinR/Inputs/WindowsEvent.cs b/TimberWinR/Inputs/WindowsEvent.cs deleted file mode 100644 index 623fbc7..0000000 --- a/TimberWinR/Inputs/WindowsEvent.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Xml.Linq; -using Microsoft.SqlServer.Server; - -namespace TimberWinR.Inputs -{ - public class WindowsEvent : InputBase - { - public const string ParentTagName = "WindowsEvents"; - public new const string TagName = "Event"; - - public string Source { get; private set; } - public bool FullText { get; private set; } - public bool ResolveSIDS { get; private set; } - public bool FormatMsg { get; private set; } - public string MsgErrorMode { get; private set; } - public bool FullEventCode { get; private set; } - public string Direction { get; private set; } - public string StringsSep { get; private set; } - public string ICheckpoint { get; private set; } - public string BinaryFormat { get; private set; } - public List Fields { get; private set; } - - public static void Parse(List events, XElement eventElement) - { - events.Add(parseEvent(eventElement)); - } - - static WindowsEvent parseEvent(XElement e) - { - return new WindowsEvent(e); - } - - WindowsEvent(XElement parent) - { - Source = ParseRequiredStringAttribute(parent, "source"); - FullText = ParseBoolAttribute(parent, "fullText", true); - ResolveSIDS = ParseBoolAttribute(parent, "resolveSIDS", true); - FormatMsg = ParseBoolAttribute(parent, "formatMsg", true); - MsgErrorMode = ParseEnumAttribute(parent, "msgErrorMode", new string[] {"NULL", "ERROR", "MSG"}, "MSG"); - FullEventCode = ParseBoolAttribute(parent, "fullEventCode", false); ; - Direction = ParseEnumAttribute(parent, "direction", new string[] { "FW", "BW" }, "FW"); - StringsSep = ParseStringAttribute(parent, "stringsSep", "|"); - BinaryFormat = ParseEnumAttribute(parent, "binaryFormat", new string[] { "ASC", "PRINT", "HEX" }, "PRINT"); - ParseFields(parent); - } - - private void ParseFields(XElement parent) - { - Dictionary allPossibleFields = new Dictionary() - { - { "EventLog", typeof(string) }, - { "RecordNumber", typeof(int) }, - { "TimeGenerated", typeof(DateTime) }, - { "TimeWritten", typeof(DateTime) }, - { "EventID", typeof(int) }, - { "EventType", typeof(int) }, - { "EventTypeName", typeof(string) }, - { "EventCategory", typeof(int) }, - { "EventCategoryName", typeof(string) }, - { "SourceName", typeof(string) }, - { "Strings", typeof(string) }, - { "ComputerName", typeof(string) }, - { "SID", typeof(string) }, - { "Message", typeof(string) }, - { "Data", typeof(string) } - }; - - Fields = base.parseFields(parent, allPossibleFields); - } - - } -} \ No newline at end of file diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index ac95fce..37fc249 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -23,9 +23,9 @@ namespace TimberWinR.Inputs public class WindowsEvtInputListener : InputListener { private int _pollingIntervalInSeconds = 1; - private TimberWinR.Inputs.WindowsEvent _arguments; + private TimberWinR.Parser.WindowsEvent _arguments; - public WindowsEvtInputListener(TimberWinR.Inputs.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) + public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken) { _arguments = arguments; @@ -44,12 +44,12 @@ namespace TimberWinR.Inputs // Instantiate the Event Log Input Format object var iFmt = new EventLogInputFormat() { - binaryFormat = _arguments.BinaryFormat, - direction = _arguments.Direction, + binaryFormat = _arguments.BinaryFormat.ToString(), + direction = _arguments.Direction.ToString(), formatMsg = _arguments.FormatMsg, fullEventCode = _arguments.FullEventCode, fullText = _arguments.FullText, - msgErrorMode = _arguments.MsgErrorMode, + msgErrorMode = _arguments.MsgErrorMode.ToString(), stringsSep = _arguments.StringsSep, resolveSIDs = _arguments.ResolveSIDS, iCheckpoint = checkpointFileName, @@ -77,8 +77,8 @@ namespace TimberWinR.Inputs { object v = record.getValue(field.Name); - if (field.FieldType == typeof(DateTime)) - v = field.ToDateTime(v).ToUniversalTime(); + //if (field.FieldType == typeof(DateTime)) + // v = field.ToDateTime(v).ToUniversalTime(); json.Add(new JProperty(field.Name, v)); } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index fa7ba62..577bd07 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -16,7 +16,7 @@ namespace TimberWinR { public Configuration Config { get; set; } - public Manager(string configurationFile=null) + public Manager(string xmlConfigFile, string jsonConfigFile) { var loggingConfiguration = new LoggingConfiguration(); @@ -36,8 +36,10 @@ namespace TimberWinR LogManager.GetCurrentClassLogger().Info("Initialized"); + + // Read the Configuration file - Config = new Configuration(configurationFile); + Config = Configuration.FromFile(jsonConfigFile); } /// diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs index b06dec0..dd09c46 100644 --- a/TimberWinR/Outputs/Redis.cs +++ b/TimberWinR/Outputs/Redis.cs @@ -6,6 +6,7 @@ using System.Net.Sockets; using System.Text; using System.Threading; using ctstone.Redis; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using System.Threading.Tasks; @@ -97,7 +98,7 @@ namespace TimberWinR.Outputs foreach (var grok in _manager.Config.Filters) { grok.Apply(json); - } + } } // diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs new file mode 100644 index 0000000..50a8b21 --- /dev/null +++ b/TimberWinR/Parser.cs @@ -0,0 +1,342 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using Microsoft.SqlServer.Server; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; + +namespace TimberWinR.Parser +{ + public abstract class LogstashFilter + { + public abstract bool Apply(JObject json); + + protected void RenameProperty(JObject json, string oldName, string newName) + { + JToken token = json[oldName]; + if (token != null) + { + json.Remove(oldName); + json.Add(newName, token); + } + } + + protected void ReplaceProperty(JObject json, string propertyName, string propertyValue) + { + if (json[propertyName] != null) + json[propertyName] = propertyValue; + } + + + protected void AddOrModify(JObject json, string fieldName, string fieldValue) + { + if (json[fieldName] == null) + json.Add(fieldName, fieldValue); + else + json[fieldName] = fieldValue; + } + + protected string ExpandField(string fieldName, JObject json) + { + foreach (var token in json.Children()) + { + string replaceString = "%{" + token.Path + "}"; + fieldName = fieldName.Replace(replaceString, json[token.Path].ToString()); + } + return fieldName; + } + } + + [JsonObject(MemberSerialization.OptIn)] + public class Field + { + [JsonProperty(PropertyName = "name")] + public string Name { get; set; } + [JsonProperty(PropertyName = "to")] + public string To { get; set; } + [JsonProperty(PropertyName = "type")] + public string FieldType { get; set; } + + + public Type DataType + { + get { return Type.GetType(FieldType); } + } + + public Field() + { + FieldType = "string"; + } + + public Field(string name) + { + Name = name; + To = Name; + FieldType = "string"; + } + public Field(string name, string type) + { + Name = name; + if (type.ToLower() == "string") + type = "System.String"; + else if (type.ToLower() == "datetime") + type = "System.DateTime"; + else if (type.ToLower() == "int" || type.ToLower() == "integer") + type = "System.Int32"; + FieldType = type; + To = Name; + } + public Field(string name, string type, string to) + { + Name = name; + FieldType = type; + To = to; + } + } + + public class WindowsEvent + { + public enum FormatKinds + { + PRINT, ASC, HEX + }; + + public enum MessageErrorModes + { + MSG, + ERROR, + NULL + }; + + public enum DirectionKinds + { + FW, + BW + }; + + [JsonProperty(PropertyName = "source")] + public string Source { get; set; } + [JsonProperty(PropertyName = "binaryFormat")] + public FormatKinds BinaryFormat { get; set; } + [JsonProperty(PropertyName = "msgErrorMode")] + public MessageErrorModes MsgErrorMode { get; set; } + [JsonProperty(PropertyName = "direction")] + public DirectionKinds Direction { get; set; } + [JsonProperty(PropertyName = "stringsSep")] + public string StringsSep { get; set; } + [JsonProperty(PropertyName = "fullEventCode")] + public bool FullEventCode { get; set; } + [JsonProperty(PropertyName = "fullText")] + public bool FullText { get; set; } + [JsonProperty(PropertyName = "resolveSIDS")] + public bool ResolveSIDS { get; set; } + [JsonProperty(PropertyName = "fields")] + public List Fields { get; set; } + [JsonProperty(PropertyName = "formatMsg")] + public bool FormatMsg { get; set; } + + public WindowsEvent() + { + StringsSep = "|"; + FormatMsg = true; + FullText = true; + Fields = new List(); + Fields.Add(new Field("EventLog", "string")); + Fields.Add(new Field("RecordNumber", "int")); + Fields.Add(new Field("TimeGenerated", "DateTime")); + Fields.Add(new Field("TimeWritten", "DateTime")); + Fields.Add(new Field("EventID", "int")); + Fields.Add(new Field("EventType", "int")); + Fields.Add(new Field("EventTypeName", "string")); + Fields.Add(new Field("EventCategory", "int")); + Fields.Add(new Field("EventCategoryName", "string")); + Fields.Add(new Field("SourceName", "string")); + Fields.Add(new Field("Strings", "string")); + Fields.Add(new Field("ComputerName", "string")); + Fields.Add(new Field("SID", "string")); + Fields.Add(new Field("Message", "string")); + Fields.Add(new Field("Data", "string")); + } + } + + public class Log + { + [JsonProperty(PropertyName = "location")] + public string Location { get; set; } + [JsonProperty(PropertyName = "iCodepage")] + public int CodePage { get; set; } + [JsonProperty(PropertyName = "recurse")] + public int Recurse { get; set; } + [JsonProperty(PropertyName = "splitLongLines")] + public bool SplitLongLines { get; set; } + + [JsonProperty(PropertyName = "fields")] + public List Fields { get; set; } + + public Log() + { + Fields = new List(); + Fields.Add(new Field("LogFilename", "string")); + Fields.Add(new Field("Index", "integer")); + Fields.Add(new Field("Text", "string")); + } + } + + public class IISW3CLog + { + [JsonProperty(PropertyName = "name")] + public string Name { get; set; } + [JsonProperty(PropertyName = "location")] + public string Location { get; set; } + [JsonProperty(PropertyName = "iCodepage")] + public int CodePage { get; set; } + [JsonProperty(PropertyName = "recurse")] + public int Recurse { get; set; } + [JsonProperty(PropertyName = "dQuotes")] + public bool DoubleQuotes { get; set; } + [JsonProperty(PropertyName = "dirTime")] + public bool DirTime { get; private set; } + [JsonProperty(PropertyName = "consolidateLogs")] + public bool ConsolidateLogs { get; private set; } + [JsonProperty(PropertyName = "minDateMod")] + public DateTime? MinDateMod { get; private set; } + + [JsonProperty(PropertyName = "fields")] + public List Fields { get; set; } + + public IISW3CLog() + { + CodePage = -2; + Recurse = 0; + Fields = new List(); + + Fields.Add(new Field("LogFilename", "string")); + Fields.Add(new Field("LogRow", "integer" )); + Fields.Add(new Field("date", "DateTime" )); + Fields.Add(new Field("time", "DateTime" )); + Fields.Add(new Field("c-ip", "string" )); + Fields.Add(new Field("cs-username", "string" )); + Fields.Add(new Field("s-sitename", "string" )); + Fields.Add(new Field("s-computername", "integer" )); + Fields.Add(new Field("s-ip", "string" )); + Fields.Add(new Field("s-port", "integer" )); + Fields.Add(new Field("cs-method", "string" )); + Fields.Add(new Field("cs-uri-stem", "string" )); + Fields.Add(new Field("cs-uri-query", "string" )); + Fields.Add(new Field("sc-status", "integer" )); + Fields.Add(new Field("sc-substatus", "integer" )); + Fields.Add(new Field("sc-win32-status", "integer" )); + Fields.Add(new Field("sc-bytes", "integer" )); + Fields.Add(new Field("cs-bytes", "integer" )); + Fields.Add(new Field("time-taken", "integer" )); + Fields.Add(new Field("cs-version", "string" )); + Fields.Add(new Field("cs-host", "string" )); + Fields.Add(new Field("cs(User-Agent)", "string" )); + Fields.Add(new Field("cs(Cookie)", "string" )); + Fields.Add(new Field("cs(Referer)", "string" )); + Fields.Add(new Field("s-event", "string" )); + Fields.Add(new Field("s-process-type", "string" )); + Fields.Add(new Field("s-user-time", "double" )); + Fields.Add(new Field("s-kernel-time", "double" )); + Fields.Add(new Field("s-page-faults", "integer" )); + Fields.Add(new Field("s-total-procs", "integer" )); + Fields.Add(new Field("s-active-procs", "integer" )); + Fields.Add(new Field("s-stopped-procs", "integer")); + } + } + + public class InputSources + { + [JsonProperty("WindowsEvents")] + public WindowsEvent[] WindowsEvents { get; set; } + + [JsonProperty("Logs")] + public Log[] Logs { get; set; } + + [JsonProperty("IISW3CLogs")] + public IISW3CLog[] IISW3CLogs { get; set; } + } + + public partial class Grok : LogstashFilter + { + [JsonProperty("condition")] + public string Condition { get; set; } + + [JsonProperty("match")] + public string[] Match { get; set; } + + [JsonProperty("add_tag")] + public string[] AddTag { get; set; } + + [JsonProperty("add_field")] + public string[] AddField { get; set; } + } + + public class Date : LogstashFilter + { + public string field { get; set; } + public string target { get; set; } + public bool convertToUTC { get; set; } + public List Pattern { get; set; } + + public override bool Apply(JObject json) + { + return false; + } + } + + public partial class Mutate : LogstashFilter + { + [JsonProperty("rename")] + public string[] Rename { get; set; } + + [JsonProperty("replace")] + public string[] Replace { get; set; } + + [JsonProperty("split")] + public string[] Split { get; set; } + } + + public class Filter + { + [JsonProperty("grok")] + public Grok Grok { get; set; } + + [JsonProperty("mutate")] + public Mutate Mutate { get; set; } + } + + public class TimberWinR + { + [JsonProperty("Inputs")] + public InputSources Inputs { get; set; } + public List Filters { get; set; } + public LogstashFilter[] AllFilters + { + get + { + var list = new List(); + foreach (var filter in Filters) + { + foreach (var prop in filter.GetType().GetProperties()) + { + object typedFilter = filter.GetType().GetProperty(prop.Name).GetValue(filter, null); + if (typedFilter != null && typedFilter is LogstashFilter) + { + list.Add(typedFilter as LogstashFilter); + } + } + } + return list.ToArray(); + } + } + } + + public class RootObject + { + public TimberWinR TimberWinR { get; set; } + } +} diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index f7bb167..9da6995 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -70,20 +70,17 @@ - - - - + True