From 5bd3f53663b8948c5ed1473a307ecb294a6008bf Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Mon, 28 Jul 2014 13:15:07 -0400 Subject: [PATCH] Finished up DateFilter lots of cleanup --- TimberWinR.ServiceHost/Program.cs | 46 ++++---------------- TimberWinR.ServiceHost/config.json | 42 ++++++++---------- TimberWinR/Filters/DateFilter.cs | 8 ++-- TimberWinR/Inputs/IISW3CInputListener.cs | 14 +++--- TimberWinR/Inputs/InputListener.cs | 32 ++++++++++++-- TimberWinR/Inputs/TailFileInputListener.cs | 20 ++++++--- TimberWinR/Inputs/TcpInputListener.cs | 23 +++++----- TimberWinR/Inputs/WindowsEvtInputListener.cs | 24 +++++----- TimberWinR/Manager.cs | 17 +++++++- TimberWinR/Parser.cs | 23 +++++++++- 10 files changed, 139 insertions(+), 110 deletions(-) diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs index ae8f015..a79f3a2 100644 --- a/TimberWinR.ServiceHost/Program.cs +++ b/TimberWinR.ServiceHost/Program.cs @@ -22,15 +22,7 @@ namespace TimberWinR.ServiceHost private static void Main(string[] args) { Arguments arguments = new Arguments(); - - var text = "Nov 21 17:27:53"; - var pattern = "MMM dd HH:mm:ss"; - - var match = Regex.Match(text, pattern); - - Type x = Type.GetType("string"); - Type x1 = Type.GetType("System.string"); - + HostFactory.Run(hostConfigurator => { string cmdLine = Environment.CommandLine; @@ -74,7 +66,8 @@ namespace TimberWinR.ServiceHost readonly CancellationToken _cancellationToken; readonly Task _serviceTask; private readonly Arguments _args; - private TcpInputListener _nlogListener; + + private TimberWinR.Manager _manager; public TimberWinRService(Arguments args) { @@ -91,8 +84,10 @@ namespace TimberWinR.ServiceHost public void Stop() { - _cancellationTokenSource.Cancel(); - _nlogListener.Shutdown(); + _cancellationTokenSource.Cancel(); + + if (_manager != null) + _manager.Shutdown(); } /// @@ -100,32 +95,7 @@ namespace TimberWinR.ServiceHost /// private void RunService() { - TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken); - -#if false - var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken); - - _nlogListener = new TcpInputListener(_cancellationToken, 5140); - outputRedis.Connect(_nlogListener); - - foreach (Parser.IISW3CLog iisw3cConfig in manager.Config.IISW3C) - { - var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken); - outputRedis.Connect(elistner); - } - - foreach (Parser.WindowsEvent eventConfig in manager.Config.Events) - { - var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken); - outputRedis.Connect(elistner); - } - - foreach (var logConfig in manager.Config.Logs) - { - var elistner = new TailFileInputListener(logConfig, _cancellationToken); - outputRedis.Connect(elistner); - } -#endif + _manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken); } } } diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json index 0e2d4cf..302f969 100644 --- a/TimberWinR.ServiceHost/config.json +++ b/TimberWinR.ServiceHost/config.json @@ -35,20 +35,7 @@ } ] }, - "Filters": [ - { - "grok": { - "condition": "[type] == \"Win32-FileLog\"", - "match": [ - "Text", - "" - ], - "add_field": [ - "host", - "%{ComputerName}" - ] - } - }, + "Filters": [ { "grok": { "condition": "[type] == \"Win32-Eventlog\"", @@ -89,24 +76,29 @@ ] } }, - { - "mutate": { - "rename": [ - "host", "Host", - "message","Message", - "SID", "Username" - ] - } - }, { "date": { "match": [ "timestamp", "MMM d HH:mm:sss", "MMM dd HH:mm:ss" - ] + ], + "target": "UtcTimestamp", + "convertToUTC": true } - } + }, + { + "mutate": { + "_comment": "Orion Rules", + "rename": [ + "host", "Host", + "message","Message", + "type","Type", + "SID", "Username" + ] + } + }, + ] } } diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs index 139f44b..4563fce 100644 --- a/TimberWinR/Filters/DateFilter.cs +++ b/TimberWinR/Filters/DateFilter.cs @@ -71,10 +71,10 @@ namespace TimberWinR.Parser if (ConvertToUTC) ts = ts.ToUniversalTime(); - //if (json[Target] == null) - // json.Add(Target, ts); - //else - // json[Target] = ts; + if (json[Target] == null) + json.Add(Target, ts); + else + json[Target] = ts; } } } diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index a25128e..4c38191 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -33,20 +33,22 @@ namespace TimberWinR.Inputs task.Start(); } + public override void Shutdown() + { + base.Shutdown(); + } + private void IISW3CWatcher() { var oLogQuery = new LogQuery(); - - var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(), - string.Format("{0}.lpc", Guid.NewGuid().ToString())); - + var iFmt = new IISW3CLogInputFormat() { codepage = _arguments.CodePage, consolidateLogs = _arguments.ConsolidateLogs, dirTime = _arguments.DirTime, dQuotes = _arguments.DoubleQuotes, - iCheckpoint = checkpointFileName, + iCheckpoint = CheckpointFileName, recurse = _arguments.Recurse, useDoubleQuotes = _arguments.DoubleQuotes }; @@ -106,6 +108,8 @@ namespace TimberWinR.Inputs firstQuery = false; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } + + Finished(); } } } diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs index cbec092..4c3a402 100644 --- a/TimberWinR/Inputs/InputListener.cs +++ b/TimberWinR/Inputs/InputListener.cs @@ -1,4 +1,5 @@ -using System.Runtime.InteropServices; +using System.IO; +using System.Runtime.InteropServices; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; @@ -14,9 +15,14 @@ namespace TimberWinR.Inputs public event Action OnMessageRecieved; private string _computerName; private string _typeName; + public AutoResetEvent FinishedEvent { get; set; } + public string CheckpointFileName { get; set; } public InputListener(CancellationToken token, string typeName) { + CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString())); + + this.FinishedEvent = new AutoResetEvent(false); this.CancelToken = token; this._typeName = typeName; this._computerName = System.Environment.MachineName + "." + @@ -26,20 +32,40 @@ namespace TimberWinR.Inputs .ToString(); } - private void AddDefaultFileds(JObject json) + public void Finished() + { + FinishedEvent.Set(); + } + public virtual void Shutdown() + { + FinishedEvent.WaitOne(); + try + { + if (File.Exists(CheckpointFileName)) + File.Delete(CheckpointFileName); + } + catch (Exception) + { + } + } + + private void AddDefaultFields(JObject json) { if (json["type"] == null) json.Add(new JProperty("type", _typeName)); if (json["host"] == null) json.Add(new JProperty("host", _computerName)); + + if (json["@timestamp"] == null) + json.Add(new JProperty("@timestamp", DateTime.UtcNow)); } protected void ProcessJson(JObject json) { if (OnMessageRecieved != null) { - AddDefaultFileds(json); + AddDefaultFields(json); OnMessageRecieved(json); } } diff --git a/TimberWinR/Inputs/TailFileInputListener.cs b/TimberWinR/Inputs/TailFileInputListener.cs index ca58266..517891a 100644 --- a/TimberWinR/Inputs/TailFileInputListener.cs +++ b/TimberWinR/Inputs/TailFileInputListener.cs @@ -34,16 +34,18 @@ namespace TimberWinR.Inputs task.Start(); } - private void FileWatcher() - { - var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(), - string.Format("{0}.lpc", Guid.NewGuid().ToString())); + public override void Shutdown() + { + base.Shutdown(); + } + private void FileWatcher() + { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, - iCheckpoint = checkpointFileName, + iCheckpoint = CheckpointFileName, recurse = _arguments.Recurse }; @@ -87,8 +89,10 @@ namespace TimberWinR.Inputs } else json.Add(new JProperty(field.Name, v)); - } - ProcessJson(json); + } + string msg = json["Text"].ToString(); + if (!string.IsNullOrEmpty(msg)) + ProcessJson(json); } } // Close the recordset @@ -106,6 +110,8 @@ namespace TimberWinR.Inputs firstQuery = false; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } + + Finished(); } } } diff --git a/TimberWinR/Inputs/TcpInputListener.cs b/TimberWinR/Inputs/TcpInputListener.cs index a83b16f..da487aa 100644 --- a/TimberWinR/Inputs/TcpInputListener.cs +++ b/TimberWinR/Inputs/TcpInputListener.cs @@ -24,13 +24,17 @@ namespace TimberWinR.Inputs _tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port); _listenThread = new Thread(new ThreadStart(ListenForClients)); _listenThread.Start(); - } - - public void Shutdown() - { - this._tcpListener.Stop(); } + + public override void Shutdown() + { + this._tcpListener.Stop(); + Finished(); + base.Shutdown(); + } + + private void ListenForClients() { this._tcpListener.Start(); @@ -62,13 +66,7 @@ namespace TimberWinR.Inputs { var tcpClient = (TcpClient)client; NetworkStream clientStream = tcpClient.GetStream(); - - string computerName = System.Environment.MachineName + "." + - Microsoft.Win32.Registry.LocalMachine.OpenSubKey( - @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") - .GetValue("Domain", "") - .ToString(); - + var message = new byte[bufferSize]; while (!CancelToken.IsCancellationRequested) { @@ -98,6 +96,7 @@ namespace TimberWinR.Inputs ProcessJson(json); } tcpClient.Close(); + Finished(); } } } diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index 46ad52d..cf6a1af 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -24,7 +24,7 @@ namespace TimberWinR.Inputs { private int _pollingIntervalInSeconds = 1; private TimberWinR.Parser.WindowsEvent _arguments; - + public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken, "Win32-Eventlog") { @@ -34,13 +34,17 @@ namespace TimberWinR.Inputs task.Start(); } + public override void Shutdown() + { + base.Shutdown(); + + } + private void EventWatcher() { var oLogQuery = new LogQuery(); - var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(), - string.Format("{0}.lpc", Guid.NewGuid().ToString())); - + // Instantiate the Event Log Input Format object var iFmt = new EventLogInputFormat() { @@ -52,15 +56,9 @@ namespace TimberWinR.Inputs msgErrorMode = _arguments.MsgErrorMode.ToString(), stringsSep = _arguments.StringsSep, resolveSIDs = _arguments.ResolveSIDS, - iCheckpoint = checkpointFileName, + iCheckpoint = CheckpointFileName, }; - - string computerName = System.Environment.MachineName + "." + - Microsoft.Win32.Registry.LocalMachine.OpenSubKey( - @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters") - .GetValue("Domain", "") - .ToString(); - + // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Source); @@ -99,6 +97,8 @@ namespace TimberWinR.Inputs firstQuery = false; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } + + Finished(); } } } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index 142992c..032802e 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Net.Sockets; using NLog; using NLog.Config; using NLog.Targets; @@ -19,11 +20,19 @@ namespace TimberWinR { public Configuration Config { get; set; } public List Outputs { get; set; } + public List Tcps { get; set; } + public List Listeners { get; set; } + public void Shutdown() + { + foreach (InputListener listener in Listeners) + listener.Shutdown(); + } public Manager(string xmlConfigFile, string jsonConfigFile, CancellationToken cancelToken) { - Outputs = new List(); - + Outputs = new List(); + Listeners = new List(); + var loggingConfiguration = new LoggingConfiguration(); // Create our default targets @@ -57,6 +66,7 @@ namespace TimberWinR foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C) { var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken); + Listeners.Add(elistner); foreach(var output in Outputs) output.Connect(elistner); } @@ -64,6 +74,7 @@ namespace TimberWinR foreach (Parser.WindowsEvent eventConfig in Config.Events) { var elistner = new WindowsEvtInputListener(eventConfig, cancelToken); + Listeners.Add(elistner); foreach (var output in Outputs) output.Connect(elistner); } @@ -71,6 +82,7 @@ namespace TimberWinR foreach (var logConfig in Config.Logs) { var elistner = new TailFileInputListener(logConfig, cancelToken); + Listeners.Add(elistner); foreach (var output in Outputs) output.Connect(elistner); } @@ -78,6 +90,7 @@ namespace TimberWinR foreach (var tcp in Config.Tcps) { var elistner = new TcpInputListener(cancelToken, tcp.Port); + Listeners.Add(elistner); foreach (var output in Outputs) output.Connect(elistner); } diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index d869348..302f690 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -455,11 +455,27 @@ namespace TimberWinR.Parser public partial class DateFilter : LogstashFilter { + public class DateFilterMatchException : Exception + { + public DateFilterMatchException() + : base("Date filter missing required match, must be 2 array entries.") + { + } + } + + public class DateFilterTargetException : Exception + { + public DateFilterTargetException() + : base("Date filter missing target") + { + } + } + [JsonProperty("match")] public string[] Match { get; set; } [JsonProperty("target")] - public string[] Target { get; set; } + public string Target { get; set; } [JsonProperty("convertToUTC")] public bool ConvertToUTC { get; set; } @@ -469,9 +485,12 @@ namespace TimberWinR.Parser public override void Validate() { + if (Match == null || Match.Length < 2) + throw new DateFilterMatchException(); + if (string.IsNullOrEmpty(Target)) + throw new DateFilterTargetException(); } - } public partial class Mutate : LogstashFilter